A platform for community discussion. Free, open, simple.
MT: Extract post references with the real Discourse markdown engine (#42990)
Gerhard Schlager9cd862e10a90
diff --git a/.github/workflows/migration-tests.yml b/.github/workflows/migration-tests.ymlindex 65941b2885d..945f328e65c 100644--- a/.github/workflows/migration-tests.yml+++ b/.github/workflows/migration-tests.yml@@ -36,6 +36,12 @@ jobs:with:fetch-depth: 1+ # The markdown engine specs build the parser bundle, which needs the+ # checkout's frontend dependencies (vendored libraries, the+ # pretty-text-processor build) - node, not Rails.+ - name: pnpm install+ run: pnpm install --frozen-lockfile+- name: RSpec (isolated, no Rails)# Each gem uses its own Gemfile (not the app's), so Rails is not on the# load path. This proves the gems work standalone.diff --git a/Gemfile.lock b/Gemfile.lockindex bdce811b82d..94ff08c8d67 100644--- a/Gemfile.lock+++ b/Gemfile.lock@@ -4,9 +4,11 @@ PATHmigrations-converters (0.0.1)activesupportcolored2+ discourse-emojisi18nmarkbridge (>= 0.4.0)migrations-core+ mini_racerpgzeitwerkdiff --git a/lib/pretty_text.rb b/lib/pretty_text.rbindex 7340cc10c87..0d24b52a31b 100644--- a/lib/pretty_text.rb+++ b/lib/pretty_text.rb@@ -38,41 +38,10 @@ module PrettyTextctx.eval(transpiled, filename: module_name)end- # The only modules from the `discourse` package which may be bundled into the- # server-side renderer. Additions must not transitively depend on- # browser-only APIs.- BUNDLED_DISCOURSE_MODULES = %w[- deprecation-workflow- lib/avatar-utils- lib/case-converter- lib/escape- lib/get-url- lib/object- loader- static/markdown-it/features- ]-- CORE_BUNDLE =- PrecompiledBundle.new(- dir: "tmp/pretty-text-processor",- filename_prefix: "pretty-text",- dependency_globs:- %w[- node_modules/.pnpm/lock.yaml- frontend/pretty-text-processor/**/*.{js,mjs,cjs,json}- frontend/pretty-text/addon/**/*.js- frontend/discourse-markdown-it/src/**/*.js- ] + BUNDLED_DISCOURSE_MODULES.map { "frontend/discourse/app/#{it}.js" },- ) do- Discourse::Utils.execute_command(- "pnpm",- "-C=frontend/pretty-text-processor",- "node",- "build.mjs",- "--discourse-modules=#{BUNDLED_DISCOURSE_MODULES.join(",")}",- chdir: Rails.root.to_s,- )- end+ # Defined in `pretty_text/core_bundle.rb`: the bundle definition has to stay+ # loadable outside a booted application, `PrettyText` does not.+ BUNDLED_DISCOURSE_MODULES = CoreBundle::DISCOURSE_MODULES+ CORE_BUNDLE = CoreBundle::BUNDLEdef self.load_or_build_core_bundleCORE_BUNDLE.load_or_builddiff --git a/lib/pretty_text/core_bundle.rb b/lib/pretty_text/core_bundle.rbnew file mode 100644index 00000000000..963dfb1ac6b--- /dev/null+++ b/lib/pretty_text/core_bundle.rb@@ -0,0 +1,44 @@+# frozen_string_literal: true++module PrettyText+ # The server-side markdown bundle, in a file of its own so that code running+ # the markdown pipeline outside a booted application can require it directly —+ # `PrettyText` itself is not loadable there.+ module CoreBundle+ # The only modules from the `discourse` package which may be bundled into+ # the server-side renderer. Additions must not transitively depend on+ # browser-only APIs.+ DISCOURSE_MODULES = %w[+ deprecation-workflow+ lib/avatar-utils+ lib/case-converter+ lib/escape+ lib/get-url+ lib/object+ loader+ static/markdown-it/features+ ]++ BUNDLE =+ PrecompiledBundle.new(+ dir: "tmp/pretty-text-processor",+ filename_prefix: "pretty-text",+ dependency_globs:+ %w[+ node_modules/.pnpm/lock.yaml+ frontend/pretty-text-processor/**/*.{js,mjs,cjs,json}+ frontend/pretty-text/addon/**/*.js+ frontend/discourse-markdown-it/src/**/*.js+ ] + DISCOURSE_MODULES.map { "frontend/discourse/app/#{it}.js" },+ ) do+ Discourse::Utils.execute_command(+ "pnpm",+ "-C=frontend/pretty-text-processor",+ "node",+ "build.mjs",+ "--discourse-modules=#{DISCOURSE_MODULES.join(",")}",+ chdir: Rails.root.to_s,+ )+ end+ end+enddiff --git a/migrations/.rubocop.yml b/migrations/.rubocop.ymlindex 8bef8c64764..27cf65d769a 100644--- a/migrations/.rubocop.yml+++ b/migrations/.rubocop.yml@@ -12,6 +12,16 @@ Style/EndlessMethod:Style/Send:Enabled: true+# Autocorrects `unless x.include?(…)` to ActiveSupport's `exclude?`, which exists+# only on the types ActiveSupport patches. These gems pass around plain classes+# carrying their own `include?` (CompactStringSet), so the rewrite turns working+# code into a NoMethodError that lint still reports as clean. Dropping just that+# pair is not possible — RuboCop merges a cop's hash parameters with the+# inherited ones instead of replacing them — and the cop is `Safe: false`+# anyway, so it is off here.+Style/InvertibleUnlessCondition:+ Enabled: false+# Constants (and classes) declared inside example groups are global — they leak# across spec files and collide load-order-dependently. Use `let` for aliases# and top-level (or stubbed) classes for test doubles.diff --git a/migrations/converters/lib/migrations/converters.rb b/migrations/converters/lib/migrations/converters.rbindex 2a2ac968b3d..ce906e7b33b 100644--- a/migrations/converters/lib/migrations/converters.rb+++ b/migrations/converters/lib/migrations/converters.rb@@ -6,7 +6,7 @@ module Migrationsmodule Converters# Directories under the gem's converters root that are framework# infrastructure rather than converter implementations.- NON_CONVERTER_DIRS = %w[adapter cli].freeze+ NON_CONVERTER_DIRS = %w[adapter cli markdown_engine].freezedef self.root_path@root_path ||= File.expand_path("../..", __dir__)@@ -80,9 +80,16 @@ module Migrations# `Migrations::Converters::Discourse::Users`. This is required by# `Migrations::Conversion::Base#steps`, which discovers steps via the# converter module's constants.+ #+ # A directory with a same-named `.rb` sibling is an explicit namespace+ # (e.g. `discourse/markdown_scanner/` + `markdown_scanner.rb`) and keeps+ # its nesting, so a converter can group a larger component into files.all.each_value do |converter_path|Dir[File.join(converter_path, "**", "*")].each do |subdir|- loader.collapse(subdir) if File.directory?(subdir)+ next unless File.directory?(subdir)+ next if File.exist?("#{subdir}.rb")++ loader.collapse(subdir)endenddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner.rbnew file mode 100644index 00000000000..1048186c7d9--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner.rb@@ -0,0 +1,97 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ # Extraction of specific constructs (uploads, internal links, quote+ # references, mentions, hashtags, custom emoji) from Discourse Markdown,+ # leaving everything else untouched — including anything inside code,+ # whether fenced, indented, inline, a `[code]` block or a `<pre>` block.+ #+ # The division of labor: the discourse-markdown-it engine decides what the+ # markdown means — what is code, what is a live mention or link. The+ # {EngineScanner} recovers which raw bytes produced each engine token. The+ # {Constructs} do not parse markdown context; they hold the source's name+ # sets, parse the migration-specific syntax (quote headers, upload ids,+ # internal routes) and build the reference objects the importer resolves.+ # The {TierGate} skips bodies with nothing extractable before any of this+ # runs.+ #+ # ## Why counting is sound+ #+ # The engine's inline tokens have no source offsets, so positions are+ # recovered by counting raw occurrences of a token's value. Counting is+ # dumb on purpose — the bytes alone, with no boundary rule and no name+ # grammar. Every live construct spells its value in the raw and one raw+ # occurrence yields at most one token, so the raw count is never below the+ # token count, and equality then forces every occurrence to be live. Any+ # rule that could reject a raw occurrence — a boundary condition, a name+ # grammar, "this looks like a longer URL" — can only lower the count, and+ # the freed token then gets matched to a look-alike inside a code span+ # instead, which corrupts the body.+ #+ # A link that resolves through a reference definition spells no URL where+ # it stands, so the engine reports that destination once, from the+ # definition's own line, and the premise holds there too.+ #+ # Two conditions are allowed anyway, because they protect the premise that+ # the bytes are there at all:+ #+ # * a body with a construct-capable character entity refuses, since+ # `@bob` yields a token whose bytes appear nowhere;+ # * names are counted folded, at least as coarsely as the engine folds+ # them ({FoldedText}). Both sides fold through+ # `FoldedText.fold`, which also merges the two lowercase+ # sigmas: JavaScript's `toLowerCase` and Ruby's `downcase` disagree on+ # a word-final `Σ`, and a fold coarser than the engine's is what the+ # argument above needs. This comparison only locates occurrences;+ # identity lookups and embed recording preserve the distinction.+ #+ # Two byte-level exclusions drop URL hits anyway, because no live link+ # can spell them:+ #+ # * a hit of a schemeless reading right after `//` is the tail of some+ # scheme-ful spelling, since linkify starts no bare-domain link+ # there;+ # * a hit that lies strictly inside a hit of another tracked value is+ # part of that longer value's byte run, so a live link there spells+ # the longer value — and where the run is shielded, both hits are.+ # A value the engine counted inside a link label is exempt, because+ # there the shorter value really does occur inside the longer one's+ # link.+ #+ # Counts that cannot be matched escalate to+ # {EngineScanner::SubstitutionPass}, which asks the engine about one+ # occurrence at a time.+ #+ # ## Refuse, never corrupt+ #+ # Extraction can refuse a construct it cannot confirm, but it cannot+ # corrupt one. Anything unconfirmed stays unchanged, and the body is+ # counted with its cause on `RawExtractor#engine_refusals` — the+ # conversion's must-resolve list. A confirmed occurrence is recorded from+ # the author's own raw bytes, never from the engine's normalized spelling.+ #+ # Accepted gaps:+ #+ # * A raw URL spelling that none of the readings reconstructs —+ # percent-encoding with lowercase hex digits, say, which the engine's+ # href normalizes to uppercase — is not counted, so the body refuses+ # instead of being rewritten.+ # * A name that also occurs inside a longer one — `@bob` in `@bobby` —+ # counts one occurrence too many and escalates. A body with hundreds+ # of such pairs runs out of substitution budget and keeps its tail+ # verbatim.+ # * A custom emoji name outside {Constructs::Emoji}'s presence shape+ # never reaches the engine tier at all: the gate cannot see it, so+ # nothing is extracted and nothing is reported.+ # * A mail gateway's link-scanner wrapper around an upload URL+ # (`…/__https:/host/secure-uploads/…__;!!…$`) is recorded as an+ # upload on the gateway's host. The importer maps a foreign host's+ # uploads only when the operator trusts that host, so the wrapper+ # stays verbatim and shows up on the import report instead.+ module MarkdownScanner+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs.rbnew file mode 100644index 00000000000..0edc6d2bf44--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs.rb@@ -0,0 +1,46 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ # One class per construct kind, tried in priority order at each trigger+ # character. Each returns a {Match} or nil.+ module Constructs+ # The byte span a detected construct covers, and the AST node+ # describing it.+ Match = Data.define(:start_pos, :end_pos, :node)++ # The part of a construct's contract that is independent of the syntax+ # it parses: the {TierGate}'s presence hook, and the byte-offset+ # matching every grammar here is written for. {Base} adds the+ # inline-syntax contract (`TRIGGERS` and `#detect`) on top; a+ # construct that parses a block opener instead — the {Quote} header —+ # mixes in only this much.+ module Detection+ # A regexp the {TierGate} adds to its presence check. Only a class+ # whose matches don't always contain one of the gate's base presence+ # characters needs one; nil (the default) means the built-in check+ # covers it.+ #+ # @return [Regexp, nil]+ def presence_pattern+ nil+ end++ private++ # Anchored match at a byte offset: every PATTERN here is+ # `\G`-anchored, so `byteindex` matches at `pos` or not at all — the+ # byte-offset analogue of `PATTERN.match(input, pos)`, but+ # positioned in O(1) however many multibyte characters precede+ # `pos`.+ def match_at(pattern, input, pos)+ input.byteindex(pattern, pos) && Regexp.last_match+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/base.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/base.rbnew file mode 100644index 00000000000..edcfcb8ee65--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/base.rb@@ -0,0 +1,140 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ module Constructs+ # Base class for the constructs that parse inline syntax. The boundary+ # look-arounds they share live in {Boundaries} and the+ # syntax-independent part of the contract in {Detection}; the+ # constants here are the shared pattern building blocks.+ class Base+ include Boundaries+ include Detection++ # The characters this construct can match at. The scanner dispatches+ # by character, so a position only runs the constructs that can+ # match there.+ #+ # @return [Array<String>]+ def triggers+ self.class::TRIGGERS+ end++ # `pos` is a byte offset into `input`, and `byte` is+ # `input.getbyte(pos)`, already read by the caller — always the+ # ordinal of one of {#triggers}.+ #+ # @return [Match, nil]+ def detect(input, pos, byte)+ raise NotImplementedError, "#{self.class} must implement #detect"+ end++ private++ # Same normalization the importer applies when it resolves the name,+ # so gate and resolution cannot disagree.+ def normalize(name)+ Migrations::NameNormalizer.normalize(name)+ end++ # The shared bare-URL detection. A normal `[text](url)` is consumed+ # whole at its `[` trigger, so an inner URL is only reached when the+ # outer bracket wasn't a handled link — a nested image+ # `[](url)`, an old lightbox, or a destination core rejected+ # itself (`[x](https://host /categories)`, which linkify then turns+ # into a bare link to the host). Rewriting the URL in place is what+ # we want in all of them. A relative URL is a link only at a `](…)`+ # target; bare in prose it stays plain text once cooked, so+ # rewriting it there would turn text into a link.+ #+ # `pattern` is the construct's own `\G`-anchored bare-URL grammar;+ # the block turns an admitted MatchData into a {Match}.+ def detect_bare_url(input, pos, pattern)+ return nil unless linkify_boundary_before?(input, pos)++ match = match_at(pattern, input, pos)+ return nil unless match+ return nil if inadmissible_protocol_relative?(input, pos, match[0])+ return nil if relative_url?(match[0]) && !link_target_boundary_before?(input, pos)++ yield match+ end++ # `//host` is protocol-relative (absolute) and `https://…` schemed.+ def relative_url?(url)+ url.start_with?("/") && !url.start_with?("//")+ end++ # A record id: at most 18 digits. Ids are stored as SQLite signed+ # 64-bit integers, and a 19-digit run overflows that range (binding+ # the bignum raises); a longer run names no record anyway.+ # Unanchored, so it composes into the route patterns; every use site+ # anchors it or guards it with a lookahead, so an overlong run fails+ # entirely instead of matching an 18-digit prefix.+ ID_PATTERN = /\d{1,18}/++ # The characters that terminate a URL body: whitespace, `)` (which+ # closes a markdown link), and the quotes and angle brackets that+ # delimit a bare URL. This is the inner negated set, so+ # `[^#{URL_TERMINATORS}]` is a URL-body character.+ URL_TERMINATORS = "\\s)\"'<>"++ # Linkify takes a few trailing bytes into a bare URL's href that no+ # URL grammar accepts: a stray `-` or `#`, sentence punctuation, a+ # non-ASCII character glued to the URL. A construct match may leave+ # at most this many wordless bytes of a confirmed URL occurrence+ # uncovered; a longer tail means the match stopped inside the URL+ # proper, and replacing a prefix would leave the rest behind as+ # literal text.+ MAX_SWALLOWED_TAIL_BYTES = 16++ # An ASCII word byte, for {swallowed_tail?}.+ TRAILING_WORD = /[0-9A-Za-z_]/+ private_constant :TRAILING_WORD++ # Whether the bytes a match leaves uncovered are such a linkify+ # tail: within the cap and without a word byte. A word byte means+ # the value's URL runs past what the grammar takes, not linkify+ # junk.+ def self.swallowed_tail?(tail)+ tail.bytesize <= MAX_SWALLOWED_TAIL_BYTES && !tail.match?(TRAILING_WORD)+ end++ # Link text, with the one level of balanced brackets CommonMark+ # allows (`[see [1]](/t/5)`). The nested bracket must not itself+ # open a link or image — the `(?!\()` — so the `[` of `[](…)`+ # never matches at the outer bracket, which would consume the inner+ # construct without recording it. Failing there matches core: links+ # don't nest.+ #+ # Every quantifier is capped and the whole run is atomic: nested+ # unbounded stars made a bracket-dense body without a closing `](`+ # backtrack across every bracket split, quadratically. The run ends+ # at the first `](` either way, so atomicity changes cost, not+ # matches. The cap only bounds that cost; inline link text has no+ # length limit in CommonMark, and AI-captioned images carry alt+ # texts of several thousand bytes.+ LINK_TEXT = /(?>[^\[\]]{0,4999}(?:\[[^\[\]]{0,4999}\](?!\()[^\[\]]{0,4999}){0,32})/++ # The padding CommonMark allows inside a link's parentheses. The+ # destination and title may sit on separate lines, but not across a+ # blank one, so at most one newline.+ LINK_GAP = /[^\S\n]*\n?[^\S\n]*/++ # A link title. Kept to a single line — one spanning lines is rare,+ # and not matching it leaves the link undetected rather than wrongly+ # detected.+ LINK_TITLE = /(?:"[^"\n]*"|'[^'\n]*'|\([^()\n]*\))/++ # Everything a markdown link allows after its destination: an+ # optional title, padding, and the closing paren. Shared so every+ # construct that reads `[text](url)` accepts the same shapes.+ LINK_TAIL = /(?:#{LINK_GAP}#{LINK_TITLE})?#{LINK_GAP}\)/+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/boundaries.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/boundaries.rbnew file mode 100644index 00000000000..8a1b4d76664--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/boundaries.rb@@ -0,0 +1,135 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ module Constructs+ # The boundary look-arounds the URL constructs share: whether core+ # lets a link open at a position, given what sits next to it. Each+ # takes the input and a byte offset, tests the neighboring byte as+ # ASCII first, and falls back to the recovered character for a+ # Unicode-aware test.+ #+ # The name-gated constructs have no boundary rules at all — see+ # {MarkdownScanner}. And wherever only ASCII bytes are tested (`(`,+ # `]`, `)`, `!`), a multibyte previous character's trailing byte can+ # never equal them, so those checks need no character recovery.+ module Boundaries+ private++ # markdown-it linkifies a bare absolute schemed URL (`https://…`) in+ # prose unless the character right before its scheme is an ASCII+ # letter, digit, `+` or `\`. Two of core's engines feed this and+ # their admissions are unioned: the inline linkify rule (its scheme+ # pattern accepts a scheme after anything outside `[A-Za-z0-9.+-]`)+ # and the core linkify ruler (linkify-it, which also admits `.` and+ # `-` as Unicode punctuation). That leaves only+ # `[A-Za-z0-9+]` non-admitting, plus `\`, which markdown escapes+ # into the following character so no link forms. Verified against+ # PrettyText.+ #+ # A byte >= 0x80 is a multibyte tail, therefore outside+ # `[A-Za-z0-9+\]` and always a boundary.+ def linkify_boundary_before?(input, pos)+ return true if pos.zero?++ byte = input.getbyte(pos - 1)+ byte >= 0x80 || !linkify_non_boundary_ascii_byte?(byte)+ end++ # The ASCII characters that do NOT open a bare linkified URL:+ # letters, digits, `+`, and `\` (a markdown escape).+ def linkify_non_boundary_ascii_byte?(byte)+ (byte >= 0x30 && byte <= 0x39) || (byte >= 0x41 && byte <= 0x5a) ||+ (byte >= 0x61 && byte <= 0x7a) || byte == 0x2b || byte == 0x5c # `+` `\`+ end++ # The character set markdown-it's core-ruler linkify (linkify-it)+ # admits right before a bare protocol-relative `//host` URL: a+ # Unicode separator, punctuation or control character, or one of the+ # three separators it lists explicitly (`<`, `>`, `|`). Narrower+ # than the schemed-URL boundary above, because the inline rule that+ # widens that one only fires on `scheme://` — so a symbol like `²`+ # or `€` before a `//host` does not open a link. Verified against+ # PrettyText.+ CORE_RULER_BOUNDARY = /[\p{Z}\p{P}\p{Cc}<>|]/+ private_constant :CORE_RULER_BOUNDARY++ # Whether a bare protocol-relative `//host` URL may open at `pos`.+ # linkify-it's `//` schema rejects a `//` right after `:` or `/`+ # (the `://` tail of a scheme we already declined, or a `///` typo)+ # and excludes `_` like every linkify boundary; otherwise it admits+ # its {CORE_RULER_BOUNDARY} set.+ def protocol_relative_boundary_before?(input, pos)+ return true if pos.zero?++ byte = input.getbyte(pos - 1)+ return false if byte == 0x3a || byte == 0x2f || byte == 0x5f # `:` `/` `_`++ char = byte < 0x80 ? byte.chr : previous_char(input, pos)+ CORE_RULER_BOUNDARY.match?(char)+ end++ # A bare match beginning `//host` is protocol-relative and uses the+ # narrower {#protocol_relative_boundary_before?} set. The `/`+ # trigger fires on every `/`, including the `//` inside a+ # `https://…` the `h` trigger already declined, so this rejects that+ # scheme tail too.+ def inadmissible_protocol_relative?(input, pos, url)+ url.start_with?("//") && !protocol_relative_boundary_before?(input, pos)+ end++ # The `](…)` outer-link-target boundary: the URL sits right after a+ # `](` and a `)` closes right before the `]`. That only happens when+ # the bracket wrapped a construct that was already consumed — the+ # outer link of a nested image `[](/t/5)` or an old+ # lightbox — so the URL here is a genuine link target, which is+ # where `Base#detect_bare_url` admits a relative one.+ def link_target_boundary_before?(input, pos)+ # 0x29 = `)`+ pos >= 3 && link_destination_before?(input, pos) && input.getbyte(pos - 3) == 0x29+ end++ # A `](` right before `pos`: the destination of a link or image,+ # whose parens delimit the URL. {#link_target_boundary_before?}+ # names the narrower nested-construct shape.+ def link_destination_before?(input, pos)+ # 0x28 = `(`, 0x5d = `]`+ pos >= 2 && input.getbyte(pos - 1) == 0x28 && input.getbyte(pos - 2) == 0x5d+ end++ # A `]:` right before `pos`, past whitespace and an optional `<`:+ # the destination of a reference definition (`[label]: <url>`).+ # Asked only of an occurrence the engine already confirmed, to name+ # which construct it belongs to — an upload rather than a bare link+ # — so it takes no part in deciding how many occurrences are live.+ def definition_destination_before?(input, pos)+ pos -= 1 if pos > 0 && input.getbyte(pos - 1) == 0x3c # `<`+ # The engine permits a definition's destination on the next line.+ pos -= 1 while pos > 0 && [0x20, 0x09, 0x0a, 0x0d].include?(input.getbyte(pos - 1))+ # 0x3a = `:`, 0x5d = `]`+ pos >= 2 && input.getbyte(pos - 1) == 0x3a && input.getbyte(pos - 2) == 0x5d+ end++ def bang_before?(input, pos)+ return false if pos.zero?++ input.getbyte(pos - 1) == 0x21 # `!`+ end++ # The character ending just before `pos`, for the Unicode-aware+ # look-backs. `pos` sits on a character boundary, so a multibyte+ # previous character is the continuation bytes (`10xxxxxx`) right+ # before `pos` plus their lead byte.+ def previous_char(input, pos)+ start = pos - 1+ start -= 1 while (input.getbyte(start) & 0xC0) == 0x80+ input.byteslice(start, pos - start)+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/emoji.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/emoji.rbnew file mode 100644index 00000000000..9189b6fdbd9--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/emoji.rb@@ -0,0 +1,54 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ module Constructs+ # Custom emoji shortcodes (`:name:`), deferred only when the name is+ # one of the source's own custom emoji. Standard emoji, a stray+ # `:word:` in prose and clock times all pass through untouched.+ #+ # Whether a shortcode opens where it sits, and what a tone suffix+ # resolves to, is core's business (see {MarkdownScanner}). Core+ # lowercases a shortcode before the custom lookup, so `:MYEMOJI:` is+ # the same emoji as `:myemoji:`.+ class Emoji < Base+ TRIGGERS = [":"].freeze++ # A bare `:` is far too common for the gate's presence check, while+ # the `:name:` shape keeps plain posts skipping all work. Wider than+ # a custom emoji name (which core spells lowercase and ASCII): the+ # raw may spell the name in any case.+ PRESENCE_PATTERN = /:[[:alnum:]_+-]+:/+ private_constant :PRESENCE_PATTERN++ # @param names [Migrations::CompactStringSet] the source's custom+ # emoji names, already folded.+ def initialize(names:)+ @names = names+ end++ # The folded name set, for the {TierGate}'s candidate probe, which+ # folds the raw itself and so needs the set unmediated.+ attr_reader :names++ def presence_pattern+ PRESENCE_PATTERN+ end++ def tracked_name?(name)+ @names.include?(normalize(name))+ end++ # The node for a confirmed occurrence. The name keeps the author's+ # spelling, which the importer folds when it looks the emoji up.+ def node_for(text)+ EmojiReference.new(name: text.delete_prefix(":").delete_suffix(":"))+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/hashtag.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/hashtag.rbnew file mode 100644index 00000000000..c33142f4c2f--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/hashtag.rb@@ -0,0 +1,62 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ module Constructs+ # Discourse hashtags (`#slug`, `#parent:child`, or a forced+ # `#name::tag` / `#name::category`). The category or tag a hashtag+ # names is resolved at import (its slug or name can change), so the+ # node just carries the name and any forced type.+ #+ # Only a hashtag naming one of the source's categories or tags is+ # deferred; `PR #123` and `channel #general` stay literal text. A+ # hashtag resolving to no real category or tag renders as inert+ # `hashtag-raw` text in core, never a cooked link, so gating on the+ # source's names keeps extraction in step with what core cooks.+ #+ # Which `#word` core reads as a hashtag, and how far the slug reaches,+ # is left to core (see {MarkdownScanner}).+ class Hashtag < Base+ TRIGGERS = ["#"].freeze++ # The type a `::category` / `::tag` suffix forces, matched+ # case-insensitively because core's own matcher is.+ FORCED_TYPE = /::(?<type>category|tag)\z/i+ private_constant :FORCED_TYPE++ # @param names [Migrations::CompactStringSet] the source's category+ # slug paths and tag names, already folded. A hashtag is deferred+ # only if its folded name is in the set.+ def initialize(names:)+ @names = names+ end++ # The folded name set, for the {TierGate}'s candidate probe, which+ # folds the raw itself and so needs the set unmediated.+ attr_reader :names++ def tracked_name?(name)+ @names.include?(normalize(name))+ end++ # The node for a confirmed occurrence. The name keeps the author's+ # spelling, and the forced-type suffix is read off the raw too,+ # because core reports its slug already lowercased.+ def node_for(text)+ ref = text.delete_prefix("#")+ match = FORCED_TYPE.match(ref)+ return HashtagReference.new(name: ref, forced_type: nil) if match.nil?++ HashtagReference.new(+ name: ref[0...match.begin(0)],+ forced_type: match[:type].downcase.to_sym,+ )+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/internal_link.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/internal_link.rbnew file mode 100644index 00000000000..1ee4ccf8d62--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/internal_link.rb@@ -0,0 +1,355 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ module Constructs+ # Detects a link pointing at another record on the *same* Discourse —+ # a topic, post, user, category, tag, group or badge — so the importer+ # can rewrite it once the id/slug maps exist, either as a markdown+ # link `[text](url)` or as a bare URL (kept bare, so oneboxes keep+ # working).+ #+ # Raw HTML anchors are out of scope: markdown-it exposes an entire raw+ # tag as an opaque HTML token, not as a link with an href, so+ # rewriting one safely needs an HTML attribute parser. So is+ # `/filter?q=…`, which names its categories and tags in the query+ # string rather than the path.+ #+ # An absolute URL qualifies when its host is one the caller+ # allowlisted, scheme-insensitively, and its path sits inside that+ # host's prefix (see {UrlOrigin}). A relative URL qualifies only where+ # it is actually a link (see `Base#detect_bare_url`). A bare URL with+ # no scheme at all (`forum.example.com/t/5`), which core's linkify+ # also links, contains no character a construct can trigger on, so the+ # engine tier rewrites that form in place instead.+ class InternalLink < Base+ TRIGGERS = ["!", "[", "h", "H", "/"].freeze++ # The route segments this construct understands, shared by the+ # presence gate and the bare-URL pattern.+ ROUTE_SEGMENT = "t|p|u|users|c|category|g|groups?|tags?|badges"+ private_constant :ROUTE_SEGMENT++ # A relative link (`/t/5`) contains no character of the gate's+ # built-in presence check, so route segments contribute their own.+ ROUTE_PRESENCE = %r{/(?:#{ROUTE_SEGMENT})/}+ private_constant :ROUTE_PRESENCE++ # A URL-body character (see `Base::URL_TERMINATORS`). The trailing+ # `\w` on the bare form keeps a sentence's `.`/`,` out of the match.+ URL_BODY = /[^#{Base::URL_TERMINATORS}]/+ private_constant :URL_BODY++ # A URL-body character outside a parenthesis, for the destination+ # runs below.+ DESTINATION_BODY = /[^(#{Base::URL_TERMINATORS}]/+ private_constant :DESTINATION_BODY++ # A destination, with the one level of balanced parentheses+ # CommonMark allows (`/search?q=(oauth2_basic)%20x`). The balanced+ # pair is tried before a lone `(`, which still ends up in the+ # destination the way it did before the pair was allowed. Every run+ # is capped and atomic, so a `(` that never closes is scanned once+ # (see the ReDoS note in `Base`).+ DESTINATION =+ /(?>(?:#{DESTINATION_BODY}{1,255}|\((?>#{DESTINATION_BODY}{0,255})\)|\(){1,32})/+ private_constant :DESTINATION++ # The destination takes the padding, optional title and `<…>` form+ # CommonMark allows (see `Base::LINK_TAIL`), or an internal link+ # written with a title would keep pointing at the source site. The+ # `<…>` alternative repeats the `url` group name, which Ruby allows.+ LINK_SYNTAX =+ /\[(?<text>#{Base::LINK_TEXT})\]\(#{Base::LINK_GAP}(?:<(?<url>[^<>\n]{1,2048})>|(?<url>#{DESTINATION}))#{Base::LINK_TAIL}/+ private_constant :LINK_SYNTAX++ LINK = /\G#{LINK_SYNTAX}/+ private_constant :LINK++ # An image whose source is an internal URL — an avatar, an emoji+ # sprite, a hotlinked screenshot of another topic. Core rewrites+ # nothing about it, but its origin (and any route in its path) has+ # to move to the destination site like a link's does, so the whole+ # `` is recorded with the alt text as the label.+ IMAGE = /\G!#{LINK_SYNTAX}/+ private_constant :IMAGE++ # The bare form fires at every whitespace-preceded `h` and `/` the+ # scanner walks past, so it must reject ordinary words inside the+ # regex engine: a permissive capture-everything pattern would cost a+ # MatchData and a string per h-word of every scanned post. Each+ # branch therefore starts with something an ordinary word fails on —+ # here the `//`, since every path on a configured host belongs to+ # the forum. There need be no path at all (`https://host` is the+ # front page), so the host and everything after it are one run,+ # ending on a word character — which keeps a sentence's `.` outside+ # the URL — or on a `/`, for the root path.+ #+ # The scheme is case-insensitive because linkify-it reads it that+ # way. The insensitivity stops there: Rails routing is+ # case-sensitive, so `/T/5` is not a topic on the destination+ # either.+ ABSOLUTE = %r{(?:(?i:https?:)?//#{URL_BODY}{0,2048}[\w/])}+ private_constant :ABSOLUTE++ # A relative URL has no host to reject on, and detection is tried at+ # every `/` in prose (`and/or`, `50/50`), so a route segment is what+ # tells a link apart from a slash. The lazy `(?:/…)*?` admits a+ # subfolder install's leading segments and demands a route segment+ # after, so a plain `/` fails without the group ever expanding.+ RELATIVE =+ %r{(?:/[^/#{Base::URL_TERMINATORS}]{1,255}){0,16}?/(?:#{ROUTE_SEGMENT})/#{URL_BODY}{0,2048}\w}+ private_constant :RELATIVE++ BARE = /\G(?<url>#{ABSOLUTE}|#{RELATIVE})/+ private_constant :BARE++ # @param hosts [Hash{String => (String, nil)}] the source's own+ # hosts, each downcased and mapped to its path prefix. Empty means+ # relative-only.+ # @param base_prefix [String, nil] the current site's own path+ # prefix. It is stripped from a relative link before the route is+ # parsed.+ # @param on_foreign_host [#call, nil] called with the host when an+ # absolute URL is rejected for a foreign host but its path still+ # parses as an internal route — the "did the operator forget a+ # former domain?" signal. The foreign path is parsed as-is, so a+ # forgotten domain that served the forum under a subdirectory+ # parses no route and stays unreported.+ def initialize(hosts: {}, base_prefix: nil, on_foreign_host: nil)+ @hosts = hosts+ @base_prefix = base_prefix+ @reported_foreign_hosts = Set.new+ @on_foreign_host = on_foreign_host+ @presence_pattern = build_presence_pattern+ end++ attr_reader :presence_pattern++ def detect(input, pos, byte)+ case byte+ when 0x21 # `!`+ detect_image(input, pos)+ when 0x5b # `[`+ detect_link(input, pos)+ when 0x68, 0x48, 0x2f+ # 0x68 = `h`, 0x48 = `H`, 0x2f = `/`+ detect_bare(input, pos)+ end+ end++ # For the engine tier, which filters URL values before any construct+ # runs and so never reaches `build` for a foreign link.+ def note_foreign_url(url)+ origin = classify(url)+ note_foreign_host(origin.host, origin.rest) if origin&.foreign+ end++ # For a URL whose position the engine tier already confirmed but+ # whose surrounding bytes are the URL itself. `route_url` is the+ # engine's normalized href, so it only resolves the host and its+ # prefix — the route and the stored suffix are read from `url`, the+ # raw spelling.+ def reference_for(route_url:, url:)+ origin = classify(route_url)+ return nil if origin.nil? || origin.foreign++ rest = raw_rest(url)+ return nil if rest.nil?++ path = UrlOrigin.path_within_prefix(rest, origin.prefix)+ return nil if path.nil?++ route_or_site_node(+ url:,+ text: nil,+ path:,+ host: origin.host,+ url_offset: 0,+ label_url_offset: nil,+ )+ end++ private++ def classify(url)+ UrlOrigin.classify(url, hosts: @hosts, base_prefix: @base_prefix)+ end++ # A route-less absolute URL (`https://host/faq`) carries no route+ # segment, so without an alternative here a post holding nothing+ # else would classify `:none` and keep pointing at the old origin.+ # The hosts do the gating themselves, far more selectively than `//`+ # would.+ #+ # Each host gets its own case-insensitive alternative. Wrapping a+ # `Regexp.union` of them in one `/i` does not work: union serializes+ # what it is given as `(?-mix:…)`, turning the flag back off inside+ # its scope.+ def build_presence_pattern+ return ROUTE_PRESENCE if @hosts.empty?++ Regexp.union(ROUTE_PRESENCE, *@hosts.keys.map { |host| /#{Regexp.escape(host)}/i })+ end++ # A markdown link, unless it's the `[` of an image ``, which+ # {#detect_image} takes whole at the `!`.+ def detect_link(input, pos)+ return nil if bang_before?(input, pos)++ match = match_at(LINK, input, pos)+ return nil unless match++ build(pos, match, url: match[:url], text: match[:text])+ end++ def detect_image(input, pos)+ match = match_at(IMAGE, input, pos)+ return nil unless match++ build(pos, match, url: match[:url], text: match[:text])+ end++ # The destination's byte offset within the matched construct, and —+ # when the label spells the destination too (a `[url](url)`+ # self-link) — that spelling's offset. The importer rewrites exactly+ # these spans, so a URL the author repeated in a link title stays+ # the author's text. An ambiguous label records no label span.+ def destination_spans(pos, match, url)+ url_offset = match.byteoffset(:url).first - pos++ label_offset = nil+ # The bare pattern has no text group at all, so guard by name.+ if match.names.include?("text") && (text = match[:text])+ index = text.byteindex(url)+ if index && text.byteindex(url, index + 1).nil?+ label_offset = (match.byteoffset(:text).first - pos) + index+ end+ end++ [url_offset, label_offset]+ end++ def detect_bare(input, pos)+ detect_bare_url(input, pos, BARE) do |match|+ build(pos, match, url: match[:url], text: nil)+ end+ end++ def build(pos, match, url:, text:)+ origin = classify(url)+ return nil if origin.nil?++ if origin.foreign+ note_foreign_host(origin.host, origin.rest)+ return nil+ end++ # A sibling app's path outside the prefix stays literal — no+ # route, no `:site` rewrite, no signal.+ return nil if origin.path.nil?++ url_offset, label_url_offset = destination_spans(pos, match, url)+ node =+ route_or_site_node(+ url:,+ text:,+ path: origin.path,+ host: origin.host,+ url_offset:,+ label_url_offset:,+ )+ return nil unless node++ Match.new(start_pos: pos, end_pos: match.byteoffset(0).last, node:)+ end++ # A bare schemeless domain starts with its host, so the part from+ # the first `/`, `?` or `#` on is the path (a bare domain with no+ # path is the front page: an empty rest).+ def raw_rest(url)+ host, rest = UrlOrigin.split(url)+ return rest unless host.nil? && rest.nil?++ start = url.index(%r{[/?\#]})+ start.nil? ? "" : url[start..]+ end++ # A resolved route builds a typed target; an absolute internal URL+ # with no route builds a `:site` target, whose origin alone is+ # rewritten. Two cases stay literal: a relative route-less URL,+ # already correct on the destination, and a coordinate-shaped path+ # that failed to parse (see `RouteParser.coordinate_shaped?`).+ def route_or_site_node(url:, text:, path:, host:, url_offset:, label_url_offset:)+ if (target = RouteParser.parse(path))+ # `path` is the extracted URL's own string (character domain),+ # so the suffix is a plain character slice — only the `Match`+ # positions are byte offsets.+ target_reference(+ url:,+ text:,+ target:,+ suffix: path[target[:route_length]..],+ url_offset:,+ label_url_offset:,+ )+ elsif host && !RouteParser.coordinate_shaped?(path)+ target_reference(+ url:,+ text:,+ target: SITE_TARGET,+ suffix: path,+ url_offset:,+ label_url_offset:,+ )+ end+ end++ # A `:site` target rewrites only the origin, so it names no record.+ SITE_TARGET = {+ target_type: :site,+ target_id: nil,+ target_name: nil,+ target_topic_id: nil,+ target_post_number: nil,+ target_tag_path: nil,+ }.freeze+ private_constant :SITE_TARGET++ def target_reference(url:, text:, target:, suffix:, url_offset:, label_url_offset:)+ InternalLinkReference.new(+ url:,+ text:,+ target_type: target[:target_type],+ target_id: target[:target_id],+ target_name: target[:target_name],+ target_topic_id: target[:target_topic_id],+ target_post_number: target[:target_post_number],+ target_tag_path: target[:target_tag_path],+ target_suffix: suffix.presence,+ url_offset:,+ label_url_offset:,+ )+ end++ # A foreign host is rejected before routing (the cheap check); only+ # a caller that asked for the signal pays for the route parse that+ # tells an internal-looking self-link on an unconfigured host from+ # an ordinary external link. Once per host: a forgotten former+ # domain can appear in millions of posts, and the signal's value is+ # the host name.+ def note_foreign_host(host, rest)+ return if @on_foreign_host.nil? || @reported_foreign_hosts.include?(host)+ return unless RouteParser.parse(rest)++ @reported_foreign_hosts << host+ @on_foreign_host.call(host)+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/internal_link/route_parser.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/internal_link/route_parser.rbnew file mode 100644index 00000000000..20a3c54b6fc--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/internal_link/route_parser.rb@@ -0,0 +1,373 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ module Constructs+ class InternalLink < Base+ # The route grammar: reads a path (with its query and fragment) into+ # the target fields of the record it names. A pure function of the+ # path string — the construct has already decided the URL is+ # internal.+ class RouteParser+ # A percent-encoded byte. A Unicode username reaches the parser+ # this way: email notifications write the link encoded, and the+ # engine's normalized href is encoded even where the author typed+ # the name literally.+ PCT_BYTE = /%\h\h/+ private_constant :PCT_BYTE++ # A name character: a Unicode alphanumeric or mark, the way core's+ # `UsernameValidator` (and the markdown-it mentions rule) reads+ # one — `\p{Alnum}\p{M}` rather than ASCII-only `\w`, which would+ # cut `café` to `caf`.+ NAME_CHAR = /[\p{Alnum}\p{M}]|#{PCT_BYTE}/+ private_constant :NAME_CHAR++ # A `/u/<name>` segment: it starts with a name character or `_`,+ # its interior may also hold `.` and `-`, and it ends on a name+ # character. So `/u/bob.` leaves the `.` out while `/u/john.doe`+ # matches whole. Each separator run binds to the character behind+ # it, so a name has exactly one parse and the run is scanned once.+ WORD = /(?:#{NAME_CHAR}|_)(?:[._-]*(?:#{NAME_CHAR}))*/+ private_constant :WORD++ # A single path segment, up to the next `/`, `?` or `#`.+ SEGMENT = %r{[^/?#]+}+ private_constant :SEGMENT++ # A decoded name may hold neither path structure nor control+ # characters: only a name that could have been written literally+ # names a record.+ UNDECODABLE = %r{[\x00-\x1f/?\#]}+ private_constant :UNDECODABLE++ # Rails takes a `.json`/`.rss` format extension off a route before+ # matching it, so an id run ends there and the extension rides in+ # the suffix. Letters only — `/t/slug/5.2` names no route on the+ # destination either — but a bare trailing dot is an empty format+ # and still routes.+ FORMAT = /\.[a-zA-Z]*/+ private_constant :FORMAT++ # Where an id ends: the path may continue, stop, or carry a format+ # extension. Nothing is consumed, so the tail becomes the suffix.+ ID_END = %r{(?=(?:#{FORMAT})?(?:[/?\#]|\z))}+ private_constant :ID_END++ # Where a route that consumed the whole path ends. A trailing `/`+ # stays outside the match, so the suffix keeps it and the rebuilt+ # URL ends the way the author wrote it.+ PATH_END = %r{(?=/?(?:[?\#]|\z))}+ private_constant :PATH_END++ # `/t/<id>` or `/t/<id>/<post_number>`, the id-only forms.+ TOPIC_NUMERIC =+ %r{\A/t/(?<id>#{Base::ID_PATTERN})(?:/(?<post_number>#{Base::ID_PATTERN}))?#{ID_END}}+ private_constant :TOPIC_NUMERIC++ # `/t/<slug>/<id>` or `/t/<slug>/<id>/<post_number>`. The slug is+ # `-` for the slugless form.+ TOPIC_SLUG =+ %r{\A/t/#{SEGMENT}/(?<id>#{Base::ID_PATTERN})(?:/(?<post_number>#{Base::ID_PATTERN}))?#{ID_END}}+ private_constant :TOPIC_SLUG++ # `/t/<slug>` alone — core routes it to the topic with that slug.+ # One segment only, and never all digits: a short digit run is an+ # id, an overlong one is the id-or-numeric-title ambiguity,+ # refused rather than guessed.+ TOPIC_SLUG_ONLY = %r{\A/t/(?<slug>(?!\d+(?=[/?#]|\z))#{SEGMENT})#{PATH_END}}+ private_constant :TOPIC_SLUG_ONLY++ POST = %r{\A/p/(?<id>#{Base::ID_PATTERN})#{ID_END}}+ private_constant :POST++ # The boundary keeps `/u/bob!!!` from reading as user `bob` — such+ # a URL is no route on the destination either, so rewriting it+ # would launder an invalid link. A dotted suffix (`/u/bob.json`)+ # folds into the name, since the name grammar allows interior+ # dots.+ USER = %r{\A/u(?:sers)?/(?<name>#{WORD})(?=[/?#]|\z)}+ private_constant :USER++ # The list filters a category's tabs are addressed by (core's+ # `Discourse.filters`).+ CATEGORY_FILTER = "latest|unread|new|unseen|top|read|posted|bookmarks|hot"+ private_constant :CATEGORY_FILTER++ # Where a category's slug path stops. Core routes all of these+ # *inside* `c/*category_slug_path_with_id`, so Rails takes the+ # tail off before the glob resolves and+ # `Category.find_by_slug_path_with_id` never sees it. The longer+ # forms all open with one of these segments.+ CATEGORY_TAIL = %r{/(?:l/(?:#{CATEGORY_FILTER})|none|all|subcategories)(?=[/?#]|\z)}+ private_constant :CATEGORY_TAIL++ # A further slug segment, unless it opens the tail. Guards only+ # the segments after the first, so a category whose own slug is+ # `all` (`/c/all`) still reads as a slug — `none` is the one core+ # reserves.+ CATEGORY_SLUG_SEGMENT = %r{(?!#{CATEGORY_TAIL})/#{SEGMENT}}+ private_constant :CATEGORY_SLUG_SEGMENT++ # Core takes the final segment before a filter tail as the id;+ # earlier numeric segments can be category slugs.+ CATEGORY_ID =+ %r{\A/(?:c|category)/(?:(?<path>#{SEGMENT}#{CATEGORY_SLUG_SEGMENT}*)/)?(?<id>#{Base::ID_PATTERN})(?=(?:#{FORMAT})?(?:#{CATEGORY_TAIL}|/?(?:[?#]|\z)))}+ private_constant :CATEGORY_ID++ # The legacy id-less form, whose segments join with `:` into a+ # slug path. Only reached when no segment parses as an id —+ # including an overlong digit run, which is a numeric slug. Stops+ # at the filter tail like the id form, so `/c/support/l/latest`+ # names `support` and not a `support:l:latest` that resolves to+ # nothing.+ CATEGORY_SLUG =+ %r{\A/(?:c|category)/(?<path>#{SEGMENT}#{CATEGORY_SLUG_SEGMENT}*)(?=[/?#]|\z)}+ private_constant :CATEGORY_SLUG++ # `/tag/<name>` / `/tags/<name>`. The guard routes the two+ # reserved multi-tag forms to their own grammars below:+ # `/tags/c/…` and `/tags/intersection/…` name several records, not+ # a tag called `c` or `intersection`. It needs the trailing `/`,+ # because a bare `/tags/intersection` IS the page of a tag with+ # that name.+ TAG = %r{\A/tags?/(?!(?:c|intersection)/)(?<name>#{SEGMENT})}+ private_constant :TAG++ # The slug is decorative in canonical tag routes; the id wins+ # even when the slug names a different source tag.+ TAG_ID =+ %r{\A/(?:tag/|tags/(?!(?:c|intersection)/))(?:#{SEGMENT}/)?(?<id>#{Base::ID_PATTERN})#{ID_END}}+ private_constant :TAG_ID++ # Where a category+tag route may continue after the tag name.+ # Anything else — most importantly a trailing numeric segment,+ # which core's canonical route order would read as a tag id —+ # leaves the route unparsed, so the ambiguity refuses instead of+ # picking a reading.+ TAG_ROUTE_END = %r{(?:(?=/l/(?:#{CATEGORY_FILTER})(?=[/?#]|\z))|#{PATH_END})}+ private_constant :TAG_ROUTE_END++ # The tag segment of a category+tag route. All-numeric is the+ # tag-id ambiguity above, and `none`/`all` are core's reserved+ # subcategory filters.+ TAG_NAME_SEGMENT = %r{(?!(?:\d+|none|all)(?=[/?#]|\z))#{SEGMENT}}+ private_constant :TAG_NAME_SEGMENT++ # `/tags/c/<category-path>/<id>/<tag>` — topics carrying a tag+ # within a category, the category read exactly like CATEGORY_ID.+ # An optional `none`/`all` between them is core's subcategory+ # filter, kept in the tag path so the rebuilt route filters the+ # same way.+ TAG_CATEGORY_ID =+ %r{\A/tags/c/(?:(?<path>#{SEGMENT}#{CATEGORY_SLUG_SEGMENT}*?)/)?(?<id>#{Base::ID_PATTERN})/(?:(?<filter>none|all)/)?(?<tag>#{TAG_NAME_SEGMENT})#{TAG_ROUTE_END}}+ private_constant :TAG_CATEGORY_ID++ # The legacy id-less form, joining segments into a slug path like+ # CATEGORY_SLUG. The path is lazy so the last eligible segment is+ # the tag.+ TAG_CATEGORY_SLUG =+ %r{\A/tags/c/(?<path>#{SEGMENT}#{CATEGORY_SLUG_SEGMENT}*?)/(?:(?<filter>none|all)/)?(?<tag>#{TAG_NAME_SEGMENT})#{TAG_ROUTE_END}}+ private_constant :TAG_CATEGORY_SLUG++ # `/tags/intersection/<t1>/<t2>[/…]` — topics carrying every+ # listed tag. Two names minimum: core routes a single-segment form+ # as that one tag's page, which the TAG route already reads.+ TAG_INTERSECTION =+ %r{\A/tags/intersection/(?<tags>#{SEGMENT}(?:/#{SEGMENT})+)#{PATH_END}}+ private_constant :TAG_INTERSECTION++ GROUP = %r{\A/(?:g|groups?)/(?<name>#{SEGMENT})}+ private_constant :GROUP++ # A path that steps INTO a coordinate-bearing route family — the+ # family segment followed by `/` and more path. A bare family+ # segment, with or without the slash (`/u`, `/badges/`,+ # `/g/?x=1`), is that family's coordinate-free index page.+ COORDINATE_OPENER =+ %r{\A/(?:t|p|u|users|c|category|g|groups?|tags?|badges)/(?![?\#]|\z)}+ private_constant :COORDINATE_OPENER++ # `/badges/<id>` or `/badges/<id>/<slug>`; the slug is regenerated+ # at import, so the route consumes it rather than keeping it as+ # suffix.+ BADGE = %r{\A/badges/(?<id>#{Base::ID_PATTERN})(?:/#{SEGMENT})?#{ID_END}}+ private_constant :BADGE++ class << self+ # Returns the target fields plus `route_length` (how much of+ # `rest` the route consumed; the remainder is the suffix), or+ # nil for an unknown path.+ def parse(rest)+ topic_or_post(rest) || post_by_id(rest) || user(rest) || category(rest) ||+ tag_category(rest) || tag_intersection(rest) || tag(rest) || group(rest) ||+ badge(rest)+ end++ # Whether an UNPARSED path still opened a coordinate-bearing+ # route family (`/t//209` with its empty slug, `/u/bob!!!`, the+ # reserved `/tags/c/…` forms). Such a tail plausibly carries the+ # OLD site's ids, so rewriting just the origin under it would+ # point the new host at stale coordinates. Callers use this to+ # keep those paths out of the origin-only `:site` rewrite and+ # report them instead.+ def coordinate_shaped?(path)+ COORDINATE_OPENER.match?(path)+ end++ private++ def topic_or_post(rest)+ match = TOPIC_NUMERIC.match(rest) || TOPIC_SLUG.match(rest)+ if match.nil?+ match = TOPIC_SLUG_ONLY.match(rest)+ # By name, like a user route: resolution looks the slug up+ # and restores the source on no unique match.+ return match && target(match, :topic, target_name: match[:slug])+ end++ if match[:post_number]+ target(+ match,+ :post,+ target_topic_id: match[:id].to_i,+ target_post_number: match[:post_number].to_i,+ )+ else+ target(match, :topic, target_id: match[:id].to_i)+ end+ end++ def post_by_id(rest)+ match = POST.match(rest)+ match && target(match, :post, target_id: match[:id].to_i)+ end++ def user(rest)+ match = USER.match(rest)+ return nil unless match++ name = decoded_name(match[:name])+ name && target(match, :user, target_name: name)+ end++ def category(rest)+ if (match = CATEGORY_ID.match(rest))+ target(match, :category, target_id: match[:id].to_i)+ elsif (match = CATEGORY_SLUG.match(rest))+ target(match, :category, target_name: match[:path].tr("/", ":"))+ end+ end++ def tag(rest)+ if (match = TAG_ID.match(rest))+ return target(match, :tag, target_id: match[:id].to_i)+ end++ match = TAG.match(rest)+ return nil unless match++ name = decoded_name(match[:name])+ name && target(match, :tag, target_name: name)+ end++ # The tag path keeps the optional `none`/`all` filter in front+ # of the tag name, so rebuilding preserves the filter without a+ # column of its own.+ def tag_category(rest)+ match = TAG_CATEGORY_ID.match(rest) || TAG_CATEGORY_SLUG.match(rest)+ return nil unless match++ tag_name = decoded_name(match[:tag])+ return nil unless tag_name++ tag_path = [match[:filter], tag_name].compact.join("/")+ if match.names.include?("id") && match[:id]+ target(+ match,+ :category_tag,+ target_id: match[:id].to_i,+ target_tag_path: tag_path,+ )+ else+ target(+ match,+ :category_tag,+ target_name: match[:path].tr("/", ":"),+ target_tag_path: tag_path,+ )+ end+ end++ def tag_intersection(rest)+ match = TAG_INTERSECTION.match(rest)+ return nil unless match++ names = match[:tags].split("/").map { |name| decoded_name(name) }+ return nil unless names.all?++ target(match, :tag_intersection, target_tag_path: names.join("/"))+ end++ def group(rest)+ match = GROUP.match(rest)+ return nil unless match++ name = decoded_name(match[:name])+ name && target(match, :group, target_name: name)+ end++ def badge(rest)+ match = BADGE.match(rest)+ match && target(match, :badge, target_id: match[:id].to_i)+ end++ # A Unicode username, group name or tag name travels+ # percent-encoded and is stored decoded, so the name the+ # importer looks up has to be decoded here. Slugs are the+ # exception: core writes an encoded-method slug into the column+ # percent-encoded, so a topic or category slug is compared the+ # way it arrives.+ #+ # Nil for bytes that are not valid UTF-8 or that decode to+ # something no name can hold — the route then refuses instead of+ # naming a record it guessed at.+ def decoded_name(name)+ return name unless name.include?("%")++ decoded = name.b.gsub(PCT_BYTE) { |escape| escape[1, 2].hex.chr }+ decoded.force_encoding(Encoding::UTF_8)+ return nil unless decoded.valid_encoding?++ UNDECODABLE.match?(decoded) ? nil : decoded+ end++ def target(+ match,+ target_type,+ target_id: nil,+ target_name: nil,+ target_topic_id: nil,+ target_post_number: nil,+ target_tag_path: nil+ )+ {+ target_type:,+ target_id:,+ target_name:,+ target_topic_id:,+ target_post_number:,+ target_tag_path:,+ route_length: match[0].length,+ }+ end+ end+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/mention.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/mention.rbnew file mode 100644index 00000000000..c779220daa9--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/mention.rb@@ -0,0 +1,53 @@+# frozen_string_literal: true++require "markbridge"++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ module Constructs+ # User and group mentions (`@name`). The mention *type* is decided+ # later by the converter's MentionClassifier (it needs the source's+ # groups and `here_mention` setting), so the node just carries the+ # name.+ #+ # Only a mention naming one of the source's mention names (every+ # username, group name, the `here_mention` value and `all`) is+ # deferred; `@3pm` stays literal text. A mention resolving to no user+ # or group renders as an inert `<span class="mention">` in core, never+ # a cooked link, so gating on the source's names keeps extraction in+ # step with what core cooks.+ #+ # Where a mention actually opens is core's business: this class+ # mirrors no boundary rule at all (see {MarkdownScanner}).+ class Mention < Base+ TRIGGERS = ["@"].freeze++ # @param names [Migrations::CompactStringSet] the source's mention+ # names, already folded. A mention is deferred only if its folded+ # name is in the set.+ def initialize(names:)+ @names = names+ end++ # The folded name set, for the {TierGate}'s candidate probe, which+ # folds the raw itself and so needs the set unmediated.+ attr_reader :names++ def tracked_name?(name)+ @names.include?(normalize(name))+ end++ # The node for a confirmed occurrence. The name keeps the author's+ # own spelling, which is what the importer stores and falls back to+ # when it can remap nothing.+ def node_for(text)+ Markbridge::AST::Mention.new(name: text.delete_prefix("@"))+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/quote.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/quote.rbnew file mode 100644index 00000000000..565e6de7557--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/quote.rb@@ -0,0 +1,148 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ module Constructs+ # Parses the opening tag of a Discourse quote (`[quote=…]`); the body+ # and `[/quote]` stay in place, and any embeds inside the body are+ # still extracted. The {EngineScanner} calls this only where a parsed+ # quote block token shows that core actually renders the tag, so no+ # block-position or forward checks live here.+ #+ # The header is read the way core's bbcode-block.js does: unquoted+ # (`[quote=bob, post:1]`) or wrapped in any quotation-mark pair core+ # recognizes. A header whose first slot is not a name —+ # `[quote="post:5, topic:9"]` — yields no node: that slot is the+ # username to core, whatever it holds, so it cooks+ # `data-username="post:5"` and no `data-post` at all, leaving no+ # coordinates to remap.+ class Quote+ # No {Base} here: that contract belongs to the constructs the+ # scanner dispatches by trigger byte, and this one is reached only+ # at a parsed quote block token.+ include Detection++ # Case-insensitive because core's bbcode rules are: `[QUOTE=bob]`+ # renders the same block, and missing it leaves the header's source+ # numbering in the imported raw. The header class excludes the+ # newline (core parses the tag within its line) and is capped, or a+ # body full of unclosed `[quote=` openers would re-scan its whole+ # tail per opener, quadratically.+ OPENING = /\G\[quote=(?<header>[^\]\n]{0,512})\]/i+ private_constant :OPENING++ # Separates "parsed, but carries nothing remappable" (skipped) from+ # "the grammar could not take it" — the engine tier reports the+ # latter, since an unparseable header may still hold remappable+ # fields.+ def parseable_opener?(input, pos)+ !match_at(OPENING, input, pos).nil?+ end++ def detect_block_opener(input, pos)+ match = match_at(OPENING, input, pos)+ return nil unless match++ username, name, post_number, topic_id =+ parse_header(strip_quote_marks(match[:header]))+ return nil if username.nil?++ Match.new(+ start_pos: pos,+ end_pos: match.byteoffset(0).last,+ node: QuoteReference.new(username:, name:, post_number:, topic_id:),+ )+ end++ private++ # Mirrors `QUOTATION_MARKS` in discourse-markdown-it's+ # bbcode-block.js. Core strips a pair only when the header opens+ # with the opening mark and closes with THAT pair's closing mark+ # around at least one character; a mismatched or one-sided mark+ # (`[quote="bob']`) and an empty pair (`[quote=""]`) are left+ # literal. Verified against PrettyText.+ QUOTE_MARK_PAIRS = [+ %w[" "],+ %w[' '],+ %w[« »],+ %w[“ ”],+ %w[” ”],+ %w[‘ ’],+ %w[„ “],+ %w[‚ ’],+ %w[‹ ›],+ ].freeze+ private_constant :QUOTE_MARK_PAIRS++ def strip_quote_marks(header)+ QUOTE_MARK_PAIRS.each do |opening, closing|+ if header.length > opening.length + closing.length && header.start_with?(opening) &&+ header.end_with?(closing)+ return header[opening.length...(header.length - closing.length)]+ end+ end+ header+ end++ # An overlong run names no real record (see `Base::ID_PATTERN`), so+ # the part is ignored like any unrecognized one.+ POST_PART = /\Apost:(#{Base::ID_PATTERN})\z/+ private_constant :POST_PART++ TOPIC_PART = /\Atopic:(#{Base::ID_PATTERN})\z/+ private_constant :TOPIC_PART++ USERNAME_PART = /\Ausername:(.+)\z/+ private_constant :USERNAME_PART++ # `post:`/`topic:` are the source's own post number and topic id.+ #+ # With an explicit `username:` part, the bare tokens before the+ # first recognized part are the display name, joined back with a+ # comma: a name containing one splits into several parts, which core+ # (quotes.js) reassembles the same way. Without a `username:` part+ # the leading token IS the username and any comma tail is dropped,+ # as core does — Discourse omits `username:` exactly when the+ # display name equals the username.+ def parse_header(string)+ explicit_username = nil+ post_number = topic_id = nil+ name_parts = []+ leading = true++ string+ .split(",")+ .map(&:strip)+ .each do |part|+ case part+ when POST_PART+ post_number = Regexp.last_match(1).to_i+ leading = false+ when TOPIC_PART+ topic_id = Regexp.last_match(1).to_i+ leading = false+ when USERNAME_PART+ explicit_username = Regexp.last_match(1)+ leading = false+ else+ name_parts << part if leading && part.present?+ end+ end++ if explicit_username+ name = name_parts.join(", ").presence+ display_name = name if name && name != explicit_username+ [explicit_username, display_name, post_number, topic_id]+ else+ [name_parts.first, nil, post_number, topic_id]+ end+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/upload.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/upload.rbnew file mode 100644index 00000000000..815a33dcf65--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/upload.rb@@ -0,0 +1,174 @@+# frozen_string_literal: true++require "markbridge"++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ module Constructs+ # Detects Discourse upload references (`upload://` URLs):+ # `` images,+ # `[file|attachment](upload://sha1.ext) (size)` attachments,+ # `[label|meta](upload://sha1.ext)` link-position labels with pipe+ # metadata (failed image pastes), and plain+ # `[label](upload://sha1.ext)` links.+ class Upload < Base+ TRIGGERS = ["!", "["].freeze++ # Like `Base::LINK_TEXT`, but with `|` kept out of the plain runs:+ # the label ends at the pipe that starts the metadata+ # (`[Juan [John] Hernandez.pdf|attachment]`). Line ends stay in,+ # as in core: an AI-captioned image carries its caption on a line+ # of its own inside the alt.+ LABEL = /(?>[^|\[\]]{0,4999}(?:\[[^\[\]]{0,4999}\](?!\()[^|\[\]]{0,4999}){0,32})/+ private_constant :LABEL++ # The URL and dimensions classes exclude the newline (a destination+ # cannot contain an unescaped line end) and are capped.+ IMAGE_PATTERN =+ %r{\G!\[(?<alt>#{LABEL})(?:\|(?<dimensions>[^\[\]\n]{0,99}))?\]\(upload://(?<url>[^)\n]{1,512})\)}++ # The marker folds case but the scheme does not, which is core's own+ # split: `[r.pdf|ATTACHMENT](upload://…)` still cooks a link+ # carrying `data-orig-href`, while `UPLOAD://` cooks one with no+ # `data-orig-href` at all — nothing the importer could resolve.+ ATTACHMENT_PATTERN =+ %r{+ \G+ \[(?<filename>#{LABEL})\|attachment\]+ \((?-i:upload)://(?<url>[^)\n]{1,512})\)+ # `\s*` here would let the group cross a line end — even a blank+ # one — and pull a following parenthesized line into the match.+ (?:[^\S\n]*\((?<size>[^)\n]{1,99})\))?+ }xi+ # A link-position label with metadata after a pipe —+ # `[image|281x500](upload://sha1.ext)` — the leftover of a failed+ # image paste. Core renders the label literally, pipe and all, as a+ # plain link to the upload, so the upload is as real as in any other+ # form. The suffix class allows pipes, so `[a|b|c]` matches whole;+ # an exact `|attachment` marker belongs to ATTACHMENT_PATTERN above.+ LINK_PATTERN =+ %r{+ \G+ \[(?<filename>#{LABEL})\|(?!attachment\])[^\[\]\n]{0,99}\]+ \((?-i:upload)://(?<url>[^)\n]{1,512})\)+ }xi+ # A plain `[label](upload://sha1.ext)` link, which core links the+ # same way.+ PLAIN_LINK_PATTERN =+ %r{+ \G+ \[(?<filename>#{LABEL})\]+ \((?-i:upload)://(?<url>[^)\n]{1,512})\)+ }xi+ private_constant :IMAGE_PATTERN, :ATTACHMENT_PATTERN, :LINK_PATTERN, :PLAIN_LINK_PATTERN++ # The `upload://` form on its own, with no `` around it: a+ # reference definition's destination, which the engine tier+ # confirmed as an image's or a link's source.+ SHORT_URL = %r{\Aupload://(?<url>[^\s)]{1,512})\z}+ private_constant :SHORT_URL++ def detect(input, pos, byte)+ case byte+ when 0x21 # `!`+ detect_image(input, pos)+ when 0x5b # `[`+ detect_attachment(input, pos) || detect_link(input, pos) ||+ detect_plain_link(input, pos)+ end+ end++ # The node for a bare {SHORT_URL} spelling, whose whole span is the+ # URL itself — the alt text and the dimensions live in the+ # `![alt][id]` that uses the definition and stay there.+ #+ # @return [Markbridge::AST::Upload, nil]+ def reference_for(url)+ match = SHORT_URL.match(url)+ return nil unless match++ sha1, filename = parse_upload_url(match[:url])+ Markbridge::AST::Upload.new(sha1:, filename:, raw: url)+ end++ private++ def detect_image(input, pos)+ match = match_at(IMAGE_PATTERN, input, pos)+ return nil unless match++ sha1, filename = parse_upload_url(match[:url])+ alt = match[:alt]+ alt = nil if alt.empty?++ node =+ Markbridge::AST::Upload.new(+ sha1:,+ filename:,+ alt:,+ dimensions: match[:dimensions],+ raw: match[0],+ )++ Match.new(start_pos: pos, end_pos: match.byteoffset(0).last, node:)+ end++ def detect_attachment(input, pos)+ match = match_at(ATTACHMENT_PATTERN, input, pos)+ return nil unless match++ sha1, = parse_upload_url(match[:url])++ node =+ Markbridge::AST::Upload.new(+ sha1:,+ filename: match[:filename],+ type: :attachment,+ size: match[:size],+ raw: match[0],+ )++ Match.new(start_pos: pos, end_pos: match.byteoffset(0).last, node:)+ end++ # Only the sha1 and the verbatim source reach the embed row: a+ # resolution hit re-renders the attachment from the destination's+ # metadata, a miss restores the label untouched.+ def detect_link(input, pos)+ build_link(pos, match_at(LINK_PATTERN, input, pos))+ end++ def detect_plain_link(input, pos)+ build_link(pos, match_at(PLAIN_LINK_PATTERN, input, pos))+ end++ def build_link(pos, match)+ return nil unless match++ sha1, = parse_upload_url(match[:url])++ node =+ Markbridge::AST::Upload.new(+ sha1:,+ filename: match[:filename],+ type: :attachment,+ raw: match[0],+ )++ Match.new(start_pos: pos, end_pos: match.byteoffset(0).last, node:)+ end++ # URL format: `sha1.ext` or just `sha1`. The filename is the full+ # `sha1.ext` string, nil without an extension.+ def parse_upload_url(url_part)+ sha1, _, rest = url_part.partition(".")+ [sha1, rest.empty? ? nil : url_part]+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/upload_url.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/upload_url.rbnew file mode 100644index 00000000000..5c8e600b46f--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/upload_url.rb@@ -0,0 +1,258 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ module Constructs+ # Detects uploads referenced by a full URL instead of a short+ # `upload://` one — markdown images, markdown links and bare+ # whitespace-delimited URLs. Three URL shapes are supported, matching+ # what core's file stores write:+ #+ # * a long URL: an `/uploads/` or `/secure-uploads/` segment with+ # an `original/` or `optimized/` path below it, and a basename+ # starting with the upload's 40-hex sha1;+ # * an S3/CDN URL: the same storage path with no `/uploads/`+ # segment, directly under the host or below a bucket prefix.+ # Only with a host — no local store writes `/original/…`;+ # * a short-URL path: `/uploads/short-url/<token>[.ext]`, whose+ # base62 token decodes here into the same 40-hex sha1.+ #+ # The storage shape must sit in the URL's path: every path segment+ # stops at `?` and `#`, so an upload path inside a query or fragment+ # (`/redirect?to=/uploads/…`) makes no upload URL — rewriting the+ # redirect would swap the author's link for the file it points at. Any+ # other `/uploads/` path matches no shape and is left alone.+ #+ # The whole URL is taken, query and fragment included, both capped so+ # a generated URL cannot make the pattern scan without bounds. A URL+ # beyond a cap matches no shape at all and stays as written —+ # replacing only a prefix would leave its tail behind the placeholder.+ #+ # Recognition does no host allowlisting: a supported shape pointing at+ # some other site still resolves to nothing at import and comes back+ # verbatim (see `UploadUrlReference`). Bare URLs follow the same rule+ # as an internal link (see `Base#detect_bare_url`).+ class UploadUrl < Base+ TRIGGERS = ["!", "[", "h", "H", "/"].freeze++ # The sha1 is 160 bits; base62 needs at most 27 characters for that.+ MAX_SHORT_TOKEN_LENGTH = 27+ private_constant :MAX_SHORT_TOKEN_LENGTH++ # One path segment: a URL-body character (see+ # `Base::URL_TERMINATORS`) that is also none of `/`, `?` and `#`.+ SEGMENT = /[^\/?##{Base::URL_TERMINATORS}]{1,255}/+ private_constant :SEGMENT++ # The scheme is case-insensitive because linkify-it reads it that+ # way, so core links `HTTPS://…` too.+ ORIGIN = %r{(?<origin>(?i:https?:)?//#{SEGMENT})}+ private_constant :ORIGIN++ UPLOADS = %r{(?:secure-)?uploads/}+ private_constant :UPLOADS++ STORAGE = %r{(?:original|optimized)/}+ private_constant :STORAGE++ # An optional query and fragment, taken whole or not at all. The+ # lookahead after each part requires that nothing is left over+ # except sentence punctuation at a URL boundary, so an over-cap+ # query fails the whole URL instead of matching up to a mid-query+ # cut. A bare `?` or `#` with nothing after it is part of the URL+ # too — a markdown destination like `(…21.png?)` keeps it, and so+ # does the engine. The final guard backs sentence punctuation out of+ # the match's end, except at such a marker.+ QUERY_FRAGMENT =+ /+ (?:+ \?+ (?:+ [^##{Base::URL_TERMINATORS}]{1,1024} (?= [.,;:!?]* (?: [##{Base::URL_TERMINATORS}] | \z ) )+ | (?= [.,;:!?]* (?: [##{Base::URL_TERMINATORS}] | \z ) | \# )+ )+ )?+ (?:+ \#+ (?:+ [^#{Base::URL_TERMINATORS}]{1,255} (?= [.,;:!?]* (?: [#{Base::URL_TERMINATORS}] | \z ) )+ | (?= [.,;:!?]* (?: [#{Base::URL_TERMINATORS}] | \z ) )+ )+ )?+ (?: (?<! [.,;:!?] ) | (?<= [?\#] ) )+ /x+ private_constant :QUERY_FRAGMENT++ # From the storage marker down to the basename and whatever follows+ # it: partition segments, the sha1, the extension or `_WxH` suffix+ # ending on a word character (a sentence's `.` after a bare URL+ # stays out), then the query and fragment. The repeated groups are+ # atomic with a lookahead deciding where they stop, so a failing+ # candidate is scanned once.+ STORAGE_TAIL =+ %r{+ #{STORAGE}+ (?> (?: (?! \h{40}[._] ) #{SEGMENT} / ){0,16} ) # depth/partition segments+ (?<sha1> \h{40} ) (?=[._]) # sha1, then the extension or `_WxH` suffix+ [^/?##{Base::URL_TERMINATORS}]{0,255} \w+ #{QUERY_FRAGMENT}+ }x+ private_constant :STORAGE_TAIL++ # The long shape, from `/uploads/` down through the storage path.+ LONG_TAIL =+ %r{+ / #{UPLOADS}+ (?> (?: (?! #{STORAGE} ) #{SEGMENT} / ){0,16} )+ #{STORAGE_TAIL}+ }x+ private_constant :LONG_TAIL++ # The token is base62, the extension optional (core routes both+ # spellings). `(?![\w/-])` stops the match when more path follows —+ # core's short-URL route has exactly one segment after `short-url/`.+ SHORT_TAIL =+ %r{+ /uploads/short-url/+ (?<short_token>[0-9a-zA-Z]{1,#{MAX_SHORT_TOKEN_LENGTH}})+ (?:\.\w{1,15})?+ (?![\w/-])+ #{QUERY_FRAGMENT}+ }x+ private_constant :SHORT_TAIL++ # The `/uploads/` segment may be absent only in the schemed or+ # protocol-relative form — the S3/CDN shape.+ URL =+ %r{+ (?<upload_url>+ #{ORIGIN} (?> (?: / (?! #{UPLOADS} | #{STORAGE} ) #{SEGMENT} ){0,16} )+ (?: #{LONG_TAIL} | / #{STORAGE_TAIL} )+ | (?> (?: / (?! #{UPLOADS} ) #{SEGMENT} ){0,16} ) #{LONG_TAIL}+ | #{ORIGIN}? (?> (?: / (?! #{UPLOADS} ) #{SEGMENT} ){0,16} ) #{SHORT_TAIL}+ )+ }x+ private_constant :URL++ # Alt text and link text are the one shared label grammar (see+ # `Base::LINK_TEXT`), so a nested image's inner construct is+ # deferred on its own at the `!` trigger.+ IMAGE = /\G!\[#{Base::LINK_TEXT}\]\(#{Base::LINK_GAP}#{URL}#{Base::LINK_TAIL}/+ private_constant :IMAGE++ LINK = /\G\[#{Base::LINK_TEXT}\]\(#{Base::LINK_GAP}#{URL}#{Base::LINK_TAIL}/+ private_constant :LINK++ BARE = /\G#{URL}/+ private_constant :BARE++ # Mirrors core's `Base62::KEYS`. A drift spec decodes sample tokens+ # through core's `Upload.sha1_from_base62_encoded` and compares.+ BASE62_KEYS = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"+ private_constant :BASE62_KEYS++ ANCHORED_URL = /\A#{URL}/+ private_constant :ANCHORED_URL++ # The cheap presence check for the {TierGate}: every supported shape+ # carries one of these markers. A storage marker alone is not enough+ # — the S3/CDN shape needs a host and the long shape an `uploads/`+ # segment, so prose that merely mentions `original/` sends nothing+ # to the engine.+ def self.candidate?(raw)+ return true if raw.include?("uploads/short-url/")+ return false unless raw.include?("original/") || raw.include?("optimized/")++ raw.include?("//") || raw.include?("uploads/")+ end++ # The {EngineScanner}'s tracked-value filter. The match must start+ # at the value's first byte, or a URL that merely contains an upload+ # path would be tracked (the redirect case, one level up), and may+ # leave only wordless linkify junk uncovered+ # ({Base.swallowed_tail?}).+ def self.tracked_value?(value)+ return false unless candidate?(value)++ match = ANCHORED_URL.match(value)+ return false if match.nil?++ swallowed_tail?(value.byteslice(match.byteoffset(0).last..))+ end++ # Nil when the token is not a valid base62 sha1. Mirrors core's+ # `Upload.sha1_from_base62_encoded`.+ def self.sha1_from_short_token(token)+ value = 0+ token.each_char do |char|+ digit = BASE62_KEYS.index(char)+ return nil if digit.nil?++ value = value * 62 + digit+ end++ hex = value.to_s(16)+ hex.length > 40 ? nil : hex.rjust(40, "0")+ end++ def detect(input, pos, byte)+ case byte+ when 0x21 # `!`+ match_with(IMAGE, input, pos)+ when 0x5b # `[`+ match_with(LINK, input, pos)+ when 0x68, 0x48, 0x2f+ # 0x68 = `h`, 0x48 = `H`, 0x2f = `/`+ detect_bare(input, pos)+ end+ end++ # The node for a URL with no `` syntax around it — a+ # reference definition's destination, whose whole span is the URL.+ # The caller filters with {.tracked_value?}, so the match here+ # covers the value.+ #+ # @return [UploadUrlReference, nil]+ def reference_for(url)+ match = ANCHORED_URL.match(url)+ match && reference_from(match)+ end++ private++ def detect_bare(input, pos)+ detect_bare_url(input, pos, BARE) { |match| build_match(pos, match) }+ end++ def match_with(pattern, input, pos)+ match = match_at(pattern, input, pos)+ match && build_match(pos, match)+ end++ def build_match(pos, match)+ node = reference_from(match)+ node && Match.new(start_pos: pos, end_pos: match.byteoffset(0).last, node:)+ end++ # Nil when a short-URL token decodes to no sha1.+ def reference_from(match)+ sha1 = match[:sha1] || self.class.sha1_from_short_token(match[:short_token])+ return nil if sha1.nil?++ origin = match[:origin]+ host = origin && UrlOrigin.split(origin)&.first+ url = match[:upload_url]+ # For the ownership check against a configured path prefix (see+ # `UploadUrlReference`).+ rest = origin ? url.byteslice(origin.bytesize..) : url++ UploadUrlReference.new(sha1:, host:, rest:)+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/emoji_reference.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/emoji_reference.rbnew file mode 100644index 00000000000..d368107596c--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/emoji_reference.rb@@ -0,0 +1,14 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ # A deferred custom emoji (`:name:`). `name` is the shortcode without+ # the surrounding colons, in the author's own spelling; an import-time+ # conflict may rename it, so the importer rewrites it there.+ EmojiReference = Data.define(:name)+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner.rbnew file mode 100644index 00000000000..c1820faf4ac--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner.rb@@ -0,0 +1,353 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ # Locates every construct the real discourse-markdown-it parse reported+ # in a post's raw bytes.+ #+ # The engine's token maps give the line range of each block. When a+ # value occurs in that range exactly as often as the engine created+ # tokens for it, every occurrence is live and all of them are replaced;+ # none can be inside a code span or a link label, because the engine+ # creates no tokens there. When a region's counts differ, the value is+ # counted once more against the whole post — the same exact equality,+ # without relying on the engine's line attribution. Why equality is+ # enough is argued once, in {MarkdownScanner}. Counts that cannot be+ # matched go to a {SubstitutionPass}.+ #+ # A confirmed name occurrence becomes a node from its own raw bytes; a+ # confirmed URL occurrence is anchored into the construct that surrounds+ # it (its `[`, its `!`, or the bare URL itself). Both go through the+ # {Constructs}, so the recorded embeds have the same shape as everywhere+ # else in the pipeline.+ class EngineScanner+ # `output` is the post with every confirmed construct replaced (the+ # input itself when nothing was). `cause` says why at least one+ # construct stayed unconfirmed; nil when everything was placed.+ # `detail` carries the exception class name for `:engine_error`.+ # `slow_parse` marks a post that only parsed on the slow retry —+ # recovered, not refused, but it will also cook very slowly on the+ # destination site.+ Result =+ Data.define(:output, :cause, :detail, :slow_parse) do+ def initialize(output:, cause:, detail: nil, slow_parse: false)+ super+ end+ end++ # A post whose parse runs into the fast ceiling gets one retry before+ # `:engine_error`. This is a deadline for the whole retry of that one+ # post, enforced per engine call in {#engine_scan}, not a fresh+ # ceiling per call. A conversion runs once, so 30 seconds on one post+ # is acceptable, and a corpus run with a 60-second ceiling recovered+ # no additional posts.+ SLOW_TIMEOUT_MS = 30_000++ # Raised when the retry deadline has passed, or when a call was cut+ # short by the shrinking deadline ceiling. The substitution pass turns+ # it into `:substitution_budget`; it never means the engine failed.+ class RetryDeadlineError < StandardError+ end++ # Locating occurrences scans the post once per distinct value, so a+ # generated post with thousands of them would spend one Ruby scan per+ # value, and none of that time sits inside V8's timeout. Above this+ # many of either kind the post refuses with `:url_volume` or+ # `:name_volume`.+ MAX_SCANNED_VALUES = 256++ # @param slow_timeout_ms [Integer, nil] the retry ceiling for a body+ # whose parse the engine terminated; nil disables the retry.+ def initialize(+ engine:,+ constructs:,+ internal_link_hosts: {},+ internal_link_base_prefix: nil,+ slow_timeout_ms: SLOW_TIMEOUT_MS,+ &on_node+ )+ @engine = engine+ @slow_timeout_ms = slow_timeout_ms+ @scan_timeout_ms = nil+ @retry_deadline = nil+ @hosts = internal_link_hosts+ @base_prefix = internal_link_base_prefix+ @on_node = on_node++ # Without one of these the scanner would silently drop every+ # construct of that kind, so all but the emoji one are required. A+ # source with no custom emoji leaves that one out on purpose.+ @mention_construct = required(constructs, Constructs::Mention)+ @hashtag_construct = required(constructs, Constructs::Hashtag)+ @quote_construct = required(constructs, Constructs::Quote)+ @internal_link_construct = required(constructs, Constructs::InternalLink)+ @emoji_construct = constructs.find { |construct| construct.is_a?(Constructs::Emoji) }+ @upload_construct = constructs.find { |construct| construct.is_a?(Constructs::Upload) }+ @upload_url_construct =+ constructs.find { |construct| construct.is_a?(Constructs::UploadUrl) }++ # URL constructs are matched from their syntax anchor: `[`, `!`, or+ # the first byte of a bare URL.+ @url_dispatch = {}+ constructs.each do |construct|+ case construct+ when Constructs::Upload, Constructs::UploadUrl, Constructs::InternalLink+ construct.triggers.each { |char| (@url_dispatch[char.ord] ||= []) << construct }+ end+ end+ @url_dispatch.each_value(&:freeze)+ @url_dispatch.freeze+ end++ # @param input [String]+ # @param scan_data [Hash, nil] a precomputed+ # `MarkdownEngine::Context#scan` element for exactly this input's+ # bytes, as a batching caller supplies. Marker substitution parses+ # live either way.+ # @return [Result]+ def scan(input, scan_data: nil)+ @scan_timeout_ms = nil+ @retry_deadline = nil+ begin+ attempt(input, scan_data)+ rescue MiniRacer::ScriptTerminatedError, MiniRacer::RuntimeError+ raise if @slow_timeout_ms.nil?++ # A deterministic JS exception fails again quickly, so the single+ # retry stays cheap.+ @engine.reset!+ @scan_timeout_ms = @slow_timeout_ms+ @retry_deadline =+ Process.clock_gettime(Process::CLOCK_MONOTONIC) + @slow_timeout_ms / 1000.0+ attempt(input, scan_data).with(slow_parse: true)+ end+ rescue MiniRacer::ScriptTerminatedError, MiniRacer::RuntimeError => error+ # An engine failure that survived the retry costs this body only;+ # the context is rebuilt so the next body gets a working engine.+ # Process and resource failures are not rescued.+ @engine.reset!+ Result.new(output: input, cause: :engine_error, detail: error.class.name)+ rescue RetryDeadlineError+ # The substitution pass turns the deadline into its budget cause+ # itself; this only catches one that passed outside it.+ Result.new(output: input, cause: :substitution_budget)+ ensure+ @scan_timeout_ms = nil+ @retry_deadline = nil+ end++ # Every engine parse for the current body goes through here. On the+ # slow retry each call gets only the time left until the per-body+ # deadline; without that, a substitution starting with one second left+ # could still run for the full ceiling.+ def engine_scan(posts)+ timeout = @scan_timeout_ms+ if @retry_deadline+ remaining_ms =+ ((@retry_deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)) * 1000).floor+ # Declining a call with no time left keeps the scan ceiling from+ # extending past the deadline.+ raise RetryDeadlineError if remaining_ms <= 0++ timeout = timeout.nil? ? remaining_ms : [timeout, remaining_ms].min+ end+ @engine.scan(posts, timeout_ms: timeout)+ end++ # The substitution pass classifies a terminated parse as spent budget+ # while the per-body deadline is in force, not as an engine failure.+ def retry_deadline_active?+ !@retry_deadline.nil?+ end++ # A terminated isolate cannot be reused; the pass that swallowed the+ # termination asks for the rebuild here.+ def reset_engine!+ @engine.reset!+ end++ attr_reader :mention_construct,+ :hashtag_construct,+ :emoji_construct,+ :quote_construct,+ :url_dispatch,+ :on_node++ # The token filters below ask the constructs, so a filter and the node+ # built for a confirmed occurrence share one name set and one folding.+ def mention_tracked?(name)+ @mention_construct.tracked_name?(name)+ end++ def emoji_tracked?(name)+ !@emoji_construct.nil? && @emoji_construct.tracked_name?(name)+ end++ # The text to match for an engine hashtag slug; nil when the name is+ # not tracked. Core's matcher lets trailing colons into the slug and+ # its lookup drops them, so the raw text to look for drops them too. A+ # `::type` suffix takes no part in the name check.+ def hashtag_text(slug)+ ref = slug.sub(/:+\z/, "")+ return nil if ref.empty?++ name = ref.sub(/::(?:category|tag)\z/, "")+ return nil unless @hashtag_construct.tracked_name?(name)++ "##{ref}"+ end++ # A link or image value the migration remaps: an `upload://` short+ # URL, a full URL with a supported upload shape (the construct's own+ # check, so an unrelated `/uploads/` path is not tracked), an absolute+ # URL on one of the source's own hosts inside that host's prefix, or a+ # site-relative path inside the base prefix that parses as a route.+ # External links are not constructs and can never refuse a body. The+ # host and prefix reading is {UrlOrigin}, the same one the construct+ # grammar uses.+ def url_tracked?(value)+ return true if value.start_with?("upload://")+ return true if Constructs::UploadUrl.tracked_value?(value)++ host, rest = UrlOrigin.split(value)+ return false if rest.nil?++ if host+ return !UrlOrigin.path_within_prefix(rest, @hosts[host]).nil? if @hosts.key?(host)++ # An internal-looking URL on an unconfigured host may be a+ # forgotten former domain; report it once per host.+ @internal_link_construct.note_foreign_url(value)+ return false+ end++ return false unless rest.start_with?("/")++ path = UrlOrigin.path_within_prefix(rest, @base_prefix)+ return false if path.nil?++ Constructs::InternalLink::RouteParser.parse(path) ? true : false+ end++ # A path that starts a coordinate route family but parses no route+ # (`/t//209`, `/u/bob!!!`) is a broken link in the source data;+ # rewriting only its origin would carry the stale coordinates onto the+ # new host. Everything else is `:unanchored` — a real gap in the+ # construct grammar.+ def unplaced_url_cause(value)+ host, rest = UrlOrigin.split(value)+ path =+ if host+ @hosts.key?(host) ? UrlOrigin.path_within_prefix(rest, @hosts[host]) : nil+ elsif rest&.start_with?("/")+ UrlOrigin.path_within_prefix(rest, @base_prefix)+ end++ parser = Constructs::InternalLink::RouteParser+ if path && parser.coordinate_shaped?(path) && parser.parse(path).nil?+ :invalid_internal_route+ else+ :unanchored+ end+ end++ # For a confirmed URL occurrence that is its own syntax (a bare+ # schemeless domain, a reference definition's destination).+ # `route_url` is the engine's normalized href; `url` is the raw+ # spelling at the occurrence.+ def bare_url_node(route_url:, url:)+ @internal_link_construct.reference_for(route_url:, url:)+ end++ # For a reference definition's destination that is an upload rather+ # than a link — `[1]: upload://…`, the definition half of a+ # `![alt][1]` image. Nil when the source has no upload constructs+ # wired or the URL is no upload after all.+ def upload_node(url)+ if url.start_with?("upload://")+ @upload_construct&.reference_for(url)+ elsif Constructs::UploadUrl.tracked_value?(url)+ @upload_url_construct&.reference_for(url)+ end+ end++ private++ def attempt(input, scan_data)+ data = scan_data || engine_scan([{ id: nil, raw: input }]).first++ # One locator per body, shared by both passes: whatever the+ # count-matching pass built — the line index, the folded copy, the+ # occurrence lists — is exactly what the substitution pass needs+ # next.+ locator = Locator.new(self, input)+ track_url_values(locator, data)++ # The engine's maps count lines after markdown-it normalized CR+ # endings away, so a CR body goes straight to the map-free+ # substitution pass.+ cause = :cr_line_endings+ unless input.include?("\r")+ result = CountingPass.new(self, input, data, locator).result+ return result if result.cause.nil?+ # Substitution checks would pay the same per-value cost the cap+ # avoids.+ return result if result.cause == :url_volume || result.cause == :name_volume+ cause = result.cause+ end++ SubstitutionPass.new(+ self,+ input,+ data,+ locator,+ cause,+ seconds_budget: substitution_seconds_budget,+ ).result+ end++ # The cross-value nesting rule needs every tracked value of the body up+ # front, so both passes see the same spans. A value the engine counted+ # inside a link label is exempt: there the shorter value really does+ # occur inside the longer one's link twice.+ def track_url_values(locator, data)+ values = Set.new+ unfiltered = Set.new++ data["blocks"].each do |block|+ block["links"].each do |link|+ href = link["href"]+ next unless url_tracked?(href)++ values << href+ unfiltered << href if link["labelHits"] > 0+ end+ block["images"].each { |src| values << src if url_tracked?(src) }+ end++ locator.track_url_values(values, unfiltered:)+ end++ # On the slow path one parse may take tens of seconds, so the+ # substitution checks get what the slow parse left of the per-body+ # deadline — not a fresh {SLOW_TIMEOUT_MS} on top.+ def substitution_seconds_budget+ return SubstitutionPass::SUBSTITUTION_SECONDS_BUDGET if @retry_deadline.nil?++ remaining = @retry_deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)+ remaining > 0 ? remaining : 0.0+ end++ def required(constructs, klass)+ construct = constructs.find { |candidate| candidate.is_a?(klass) }+ return construct if construct++ raise ArgumentError, "the engine scanner needs a #{klass} construct"+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/counting_pass.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/counting_pass.rbnew file mode 100644index 00000000000..427a6ecf8b6--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/counting_pass.rb@@ -0,0 +1,235 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ class EngineScanner+ # The pass answers one question: does the number of raw occurrences of+ # a value equal the number of engine tokens for it, exactly. Any+ # inequality escalates to the substitution pass. It never accepts on+ # unequal counts and never decides from the shape of the bytes —+ # {MarkdownScanner} explains why that is what makes the equality mean+ # anything.+ class CountingPass+ def initialize(scanner, input, data, locator)+ @scanner = scanner+ @input = input+ @data = data+ @locator = locator+ end++ def result+ spans = {}++ cause = collect_expected+ cause ||= match_all_counts+ cause ||= resolve_urls(spans)+ cause ||= resolve_names(spans)+ cause ||= resolve_quotes(spans)+ return refusal(cause) if cause++ ordered = spans.values.sort_by(&:start_pos)+ return refusal(:overlap) if @locator.overlapping?(ordered)++ Result.new(output: ordered.empty? ? @input : @locator.splice(ordered), cause: nil)+ end++ private++ def refusal(cause)+ Result.new(output: @input, cause:)+ end++ # Groups the engine's construct values: per (kind, value) the+ # expected count in each block region and in total. Values the+ # migration does not remap (external links, unknown names, standard+ # emoji) never enter count matching. A name's key is its folded+ # spelling, so token values that fold alike — `@Bob` and `@bob` —+ # add up into one count.+ def collect_expected+ @expected = {}+ regions_with_constructs = []++ @data["blocks"].each do |block|+ # Not every inline block has a line map; table cells do not. A+ # mapless block's constructs are counted against the whole body,+ # and the entity check widens accordingly.+ range = @locator.region_range(block["map"]) || ([email protected])++ had = false+ block["mentions"].each do |content|+ next unless @scanner.mention_tracked?(content.delete_prefix("@"))+ had = true+ add_expected(:mention, FoldedText.fold(content), range, 1)+ end+ block["hashtags"].each do |hashtag|+ text = @scanner.hashtag_text(hashtag["slug"])+ next if text.nil?+ had = true+ add_expected(:hashtag, FoldedText.fold(text), range, 1)+ end+ block["emojis"].each do |name|+ next unless @scanner.emoji_tracked?(name)+ had = true+ add_expected(:emoji, FoldedText.fold(":#{name}:"), range, 1)+ end+ block["links"].each do |link|+ href = link["href"]+ next unless @scanner.url_tracked?(href)+ had = true+ add_expected(:url, href, range, 1 + link["labelHits"])+ end+ block["images"].each do |src|+ next unless @scanner.url_tracked?(src)+ had = true+ add_expected(:url, src, range, 1)+ end++ regions_with_constructs << range if had+ end++ url_values = @expected.count { |(kind, _), _| kind == :url }+ return :url_volume if url_values > MAX_SCANNED_VALUES+ return :name_volume if @expected.size - url_values > MAX_SCANNED_VALUES++ # Entities decode before the engine's text rules run, so a token+ # value may not exist as literal bytes at all.+ regions_with_constructs.uniq.each do |range|+ return :entity if @locator.entity_in?(range)+ end++ nil+ end++ def add_expected(kind, value, range, count)+ entry = @expected[[kind, value]] ||= { regions: Hash.new(0), total: 0 }+ entry[:regions][range] += count+ entry[:total] += count+ end++ # Two-stage count matching per value. A value matched inside its+ # regions replaces only those occurrences, so the same value in a+ # code fence elsewhere stays untouched; a value matched against the+ # whole body replaces every occurrence, and the entity check widens+ # accordingly.+ def match_all_counts+ whole = [email protected]+ @matched = {}++ @expected.each do |(kind, value), entry|+ occurrences = match_region_counts(kind, value, entry)++ if occurrences+ @matched[[kind, value]] = occurrences+ next+ end++ return :entity if @locator.entity_offsets.any?++ global = match_counts_in(kind, value, whole, entry[:total])+ return :count_mismatch if global.nil?++ @matched[[kind, value]] = global+ end++ nil+ end++ # Nil when any region's count does not match.+ def match_region_counts(kind, value, entry)+ occurrences = []+ entry[:regions].each do |range, expected|+ in_region = match_counts_in(kind, value, range, expected)+ return nil if in_region.nil?++ occurrences.concat(in_region)+ end+ occurrences+ end++ # Nil unless the number of occurrences equals what the engine saw.+ def match_counts_in(kind, value, range, expected)+ spans =+ if kind == :url+ @locator.url_spans(value, range)+ else+ @locator.occurrences_within(@locator.folded_occurrences(kind, value), range)+ end++ spans if spans.size == expected+ end++ # Turns matched URL occurrences into node matches that cover the+ # whole construct. A destination inside `[text](…)` must be replaced+ # together with its syntax, so the classes match from the `[` or `!`+ # anchor; that also covers both occurrences of a `[URL](same URL)`+ # self-link with one node. An occurrence that is its own syntax (a+ # bare schemeless domain, a reference definition's destination)+ # resolves through the engine's href. One that neither can take+ # refuses the body.+ def resolve_urls(spans)+ @matched.each do |(kind, value), occurrences|+ next unless kind == :url++ occurrences.each do |occurrence|+ match =+ @locator.anchor_match(occurrence) ||+ @locator.bare_value_match(value, occurrence)+ return @scanner.unplaced_url_cause(value) if match.nil?+ spans[[match.start_pos, match.end_pos]] ||= match+ end+ end++ nil+ end++ # Mentions, hashtags and emoji cover exactly their matched+ # occurrence.+ def resolve_names(spans)+ @matched.each do |(kind, _value), occurrences|+ next if kind == :url++ occurrences.each do |occurrence|+ match = @locator.node_match(kind, occurrence)+ spans[[match.start_pos, match.end_pos]] ||= match+ end+ end++ nil+ end++ # Quote openers come from block tokens: the engine reports the+ # quote's line range directly, so no counting is needed. Includes+ # the single-line `[quote=…]body[/quote]` form.+ def resolve_quotes(spans)+ @data["blockTokens"].each do |token|+ next unless token["type"] == "bbcode_open" && token["tag"] == "blockquote"++ range = @locator.region_range(token["map"])+ next if range.nil?++ line_end = @locator.line_starts[token["map"][0] + 1] || @input.bytesize+ opener = @input.byteindex(/\[quote=/i, range.begin)+ next if opener.nil? || opener >= line_end++ # A header that parses but has no username has nothing to remap;+ # core renders it without coordinates. A header the grammar+ # could not read at all may still hold remappable fields, so it+ # refuses instead of keeping stale data.+ match = @scanner.quote_construct.detect_block_opener(@input, opener)+ if match+ spans[[match.start_pos, match.end_pos]] ||= match+ elsif [email protected]_construct.parseable_opener?(@input, opener)+ return :unanchored+ end+ end++ nil+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/locator.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/locator.rbnew file mode 100644index 00000000000..68899bbc9e4--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/locator.rb@@ -0,0 +1,550 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ class EngineScanner+ # The byte-domain machinery for one body: finding a value's+ # occurrences in the raw, turning an occurrence into a match that+ # covers the whole construct, and splicing the accepted matches. One+ # locator serves both passes, so the line index, the folded copy, the+ # occurrence lists and the URL spans are built at most once.+ class Locator+ include Constructs::Boundaries++ # Where an occurrence starts and how many bytes the matched reading+ # spans there (an alternate URL reading can differ in length from+ # the engine's value).+ Occurrence = Data.define(:offset, :length)++ # One spelling of a URL value. `schemeless` marks a reading the+ # scheme was stripped from, which is the only kind linkify can turn+ # back into a link on its own. `pattern` is set where the raw may+ # spell the reading with fewer escapes than the engine's href+ # ({#url_pattern}); it subsumes the plain scan for that text.+ Reading = Data.define(:text, :schemeless, :pattern)+ private_constant :Reading++ # The characters markdown-it's destination normalization leaves+ # alone; every other one it percent-escapes.+ URL_SAFE = %r{[A-Za-z0-9;/?:@&=+$,\-_.!~*'()#]}+ private_constant :URL_SAFE++ EMPTY_OCCURRENCES = [].freeze+ private_constant :EMPTY_OCCURRENCES++ EMPTY_VALUES = Set.new.freeze+ private_constant :EMPTY_VALUES++ # How far back a link's `[`/`!` anchor is searched from its+ # destination. Far above any real label, AI captions of a few+ # thousand bytes included; it only bounds the walk on bodies with+ # no anchor to find.+ ANCHOR_WINDOW = 16_384+ private_constant :ANCHOR_WINDOW++ def initialize(scanner, input)+ @scanner = scanner+ @input = input+ @occurrences = {}+ @url_spans = {}+ @raw_url_spans = {}+ @url_readings = {}+ @pattern_spans = {}+ @tracked_url_values = nil+ @unfiltered_url_values = EMPTY_VALUES+ build_line_index+ end++ # The byte offset of every line start. The engine's maps count lines+ # after markdown-it normalized CR endings away, so only a CR-free+ # body may read regions off these.+ attr_reader :line_starts++ # Nil when the map names a line the body does not have.+ def region_range(map)+ return nil if map.nil?++ from = @line_starts[map[0]]+ return nil if from.nil?++ to = @line_starts[map[1]] || @input.bytesize+ from...to+ end++ # Every raw span that folds to `value`, for a mention, hashtag or+ # custom emoji: non-overlapping hits of the folded value in the+ # folded body, left to right, and nothing else — no boundary rule+ # and no name grammar (see {MarkdownScanner}). A leftmost scan for a+ # fixed-length needle returns a maximum set of disjoint hits, so the+ # count can only be too high. Memoized per value, one scan per+ # value.+ def folded_occurrences(kind, value)+ return EMPTY_OCCURRENCES if value.empty?++ @occurrences[[kind, value]] ||= begin+ spans = []+ pos = 0+ while (index = folded.text.byteindex(value, pos))+ span = raw_span_of(index, value.bytesize, value)+ if span+ spans << Occurrence.new(offset: span[0], length: span[1])+ pos = index + value.bytesize+ else+ # A hit that denotes no raw span is no occurrence, so the+ # next one may start inside it.+ pos = index + 1+ end+ end+ spans.freeze+ end+ end++ # The occurrences of a URL value in `range` — every reading (the+ # engine normalizes URLs: percent-encoding, linkify adding a scheme+ # to a bare-domain autolink) unioned, deduplicated, and coalesced+ # where readings of one occurrence nest.+ #+ # The readings must be counted as ONE union, never independently:+ # with `` `http://host/t/5` `` in code and the schemeless spelling+ # in prose, the scheme-ful reading alone counts 1 and would match+ # the code span — the union counts 2 against the engine's 1 and+ # refuses, so the substitution pass can confirm which span is live.+ #+ # Hits that sit inside another tracked value's occurrence are then+ # dropped ({#reject_nested}).+ #+ # Memoized per (value, range), and bounded by+ # `EngineScanner::MAX_SCANNED_VALUES`.+ #+ # @return [Array<Occurrence>] sorted by offset+ def url_spans(value, range)+ @url_spans[[value, range]] ||= reject_nested(value, raw_url_spans(value, range))+ end++ # The tracked URL values of the body, so a hit can be recognized as+ # part of another value's occurrence (see {#reject_nested}).+ # `unfiltered` names the values that keep every hit.+ def track_url_values(values, unfiltered:)+ @tracked_url_values = values+ @unfiltered_url_values = unfiltered+ end++ def occurrences_within(occurrences, range)+ occurrences.select do |occurrence|+ occurrence.offset >= range.begin &&+ occurrence.offset + occurrence.length <= range.end+ end+ end++ # One scan for the whole body, so each region precondition is a+ # binary search instead of a byteslice plus a fresh scan.+ def entity_offsets+ @entity_offsets ||= TierGate.construct_capable_entity_offsets(@input)+ end++ def entity_in?(range)+ index = entity_offsets.bsearch_index { |offset| offset >= range.begin }+ !index.nil? && entity_offsets[index] < range.end+ end++ # Built from the raw bytes, so the author's own spelling is recorded+ # (`@Bob`, `#Support::CATEGORY`, `:MYEMOJI:`) even though the engine+ # reported a folded value.+ def node_match(kind, occurrence)+ text = @input.byteslice(occurrence.offset, occurrence.length)+ Constructs::Match.new(+ start_pos: occurrence.offset,+ end_pos: occurrence.offset + occurrence.length,+ node: construct_for(kind).node_for(text),+ )+ end++ # Try, nearest first, every possible syntax anchor within+ # `ANCHOR_WINDOW` bytes before the occurrence (a label may span+ # lines, so no line bound), then the occurrence itself as a bare+ # URL.+ def anchor_match(occurrence)+ offset = occurrence.offset+ from = [offset - ANCHOR_WINDOW, 0].max++ pos = offset - 1+ while pos >= from+ byte = @input.getbyte(pos)+ if byte == 0x5b && pos > from && @input.getbyte(pos - 1) == 0x21 # `` out of ``.+ match =+ construct_match_at(pos - 1, occurrence) || construct_match_at(pos, occurrence)+ return match if match+ pos -= 2+ elsif byte == 0x5b || byte == 0x21 # `[` `!`+ match = construct_match_at(pos, occurrence)+ return match if match+ pos -= 1+ else+ pos -= 1+ end+ end++ # Linkify can take trailing bytes into the href that no URL+ # grammar accepts, so leaving exactly those bytes literal is+ # correct — core re-linkifies the whole thing after resolution.+ # `allow_prefix` bounds what may stay uncovered+ # ({Constructs::Base.swallowed_tail?}). Inside a destination's+ # parens there is no linkify tail to leave behind: the bytes the+ # grammar stopped at belong to the URL, so the whole occurrence+ # must be covered or nothing is.+ construct_match_at(+ offset,+ occurrence,+ allow_prefix: !link_destination_before?(@input, offset),+ )+ end++ # A confirmed occurrence that is its own whole construct: a bare+ # schemeless domain (linkify links it, but no construct grammar has+ # a byte to trigger on) or a reference definition's destination,+ # which the engine reports from the definition's own line. The+ # engine's href carries the scheme the route parses from; the span+ # replaced is exactly the raw spelling.+ def bare_value_match(value, occurrence)+ raw_spelling = @input.byteslice(occurrence.offset, occurrence.length)+ node =+ destination_upload_node(occurrence, raw_spelling) ||+ bare_link_node(value, occurrence, raw_spelling)+ return nil if node.nil?++ Constructs::Match.new(+ start_pos: occurrence.offset,+ end_pos: occurrence.offset + occurrence.length,+ node:,+ )+ end++ def overlapping?(ordered)+ ordered.each_cons(2).any? { |left, right| right.start_pos < left.end_pos }+ end++ def splice(ordered)+ result = +""+ pos = 0++ ordered.each do |match|+ result << @input.byteslice(pos...match.start_pos) if match.start_pos > pos+ source = @input.byteslice(match.start_pos...match.end_pos)+ result << (@scanner.on_node.call(match.node, source) || source)+ pos = match.end_pos+ end+ result << @input.byteslice(pos..) if pos < @input.bytesize++ result+ end++ private++ # A destination that is an upload defines the upload itself, so the+ # upload constructs answer for it and only the destination is+ # replaced: the `![alt][id]` that uses a definition keeps its own+ # syntax, and so does a `[](upload://x)` lightbox+ # link, whose label no grammar takes whole. The two look-backs name+ # which construct the occurrence belongs to, nothing about how many+ # occurrences are live.+ def destination_upload_node(occurrence, raw_spelling)+ unless link_destination_before?(@input, occurrence.offset) ||+ definition_destination_before?(@input, occurrence.offset)+ return nil+ end++ @scanner.upload_node(raw_spelling)+ end++ # The confirmation already answered "is this really a link here?",+ # so only the linkify opening boundary is re-checked, mirroring what+ # core requires ahead of a bare URL.+ def bare_link_node(value, occurrence, raw_spelling)+ return nil unless linkify_boundary_before?(@input, occurrence.offset)++ @scanner.bare_url_node(route_url: value, url: raw_spelling)+ end++ def build_line_index+ @line_starts = [0]+ offset = 0+ while (offset = @input.byteindex("\n", offset))+ offset += 1+ @line_starts << offset+ end+ end++ def folded+ @folded ||= FoldedText.new(@input)+ end++ # Nil when the hit denotes no raw span: one starting or ending+ # inside a grapheme cluster names no raw bytes, and the round trip+ # re-folds the raw slice so a mapping that does not reproduce the+ # value is dropped rather than trusted. Both only ever lower the+ # count, so they escalate.+ def raw_span_of(offset, length, value)+ span = folded.raw_span(offset, length)+ return nil if span.nil?+ return nil unless FoldedText.fold(@input.byteslice(span[0], span[1])) == value++ span+ end++ def construct_for(kind)+ case kind+ when :mention+ @scanner.mention_construct+ when :hashtag+ @scanner.hashtag_construct+ when :emoji+ @scanner.emoji_construct+ end+ end++ def url_readings_for(value)+ @url_readings[value] ||= url_readings(value)+ end++ # The union of the readings of one value in `range`, before the+ # cross-value pass. Memoized on its own, because that pass reads the+ # unfiltered spans of every tracked value.+ def raw_url_spans(value, range)+ @raw_url_spans[[value, range]] ||= begin+ spans = []+ url_readings_for(value).each { |reading| collect_url_spans(reading, range, spans) }+ spans.uniq!+ # Outermost first, so coalescing keeps the widest reading of a+ # nested pair.+ spans.sort_by! { |span| [span.offset, -span.length] }+ coalesce(spans).freeze+ end+ end++ def collect_url_spans(reading, range, spans)+ if reading.pattern+ pattern_spans(reading).each do |span|+ if span.offset >= range.begin && span.offset + span.length <= range.end+ spans << span+ end+ end+ return+ end++ text = reading.text+ length = text.bytesize+ return if length == 0++ pos = range.begin+ limit = range.end - length+ while pos <= limit && (index = @input.byteindex(text, pos))+ break if index > limit++ spans << Occurrence.new(offset: index, length:) unless scheme_tail?(reading, index)+ pos = index + 1+ end+ end++ # A schemeless reading right after `//` is the tail of some scheme-ful+ # spelling: linkify starts no bare-domain link there, and a+ # destination or autolink written that way would carry its scheme,+ # so no live link can spell the value at this position.+ def scheme_tail?(reading, offset)+ reading.schemeless && offset >= 2 && @input.byteslice(offset - 2, 2) == "//"+ end++ # Hits of `value` that lie strictly inside a hit of another tracked+ # value are dropped: the enclosing bytes are one contiguous URL run,+ # so a live link there spells the longer value, and where the run is+ # shielded — a code span, a link label — both hits are shielded+ # alike. Values whose engine count includes a label hit keep every+ # hit, because there the shorter value is expected twice inside the+ # longer one's link.+ def reject_nested(value, spans)+ return spans if @tracked_url_values.nil? || @unfiltered_url_values.include?(value)++ offsets, ends = enclosing_spans+ return spans if offsets.empty?++ spans.reject { |span| enclosed?(offsets, ends, span) }.freeze+ end++ # Every tracked value's hits over the whole body, as start offsets+ # sorted ascending plus the running maximum of their end offsets, so+ # a containment test is two binary searches.+ def enclosing_spans+ @enclosing_spans ||=+ begin+ whole = [email protected]+ all = @tracked_url_values.flat_map { |value| raw_url_spans(value, whole) }+ all.uniq!+ all.sort_by!(&:offset)+ furthest = 0+ ends = all.map { |span| furthest = [furthest, span.offset + span.length].max }+ [all.map(&:offset).freeze, ends.freeze]+ end+ end++ def enclosed?(offsets, ends, span)+ finish = span.offset + span.length+ # A hit starting earlier encloses as soon as it reaches as far; one+ # starting here has to reach further, or it is the hit itself.+ before = last_index_below(offsets, span.offset)+ return true if before && ends[before] >= finish++ here = last_index_below(offsets, span.offset + 1)+ !here.nil? && ends[here] > finish+ end++ def last_index_below(offsets, limit)+ index = offsets.bsearch_index { |offset| offset >= limit }+ index = offsets.size if index.nil?+ index > 0 ? index - 1 : nil+ end++ # Readings of one value can nest — the schemeless reading of a+ # scheme-ful spelling sits inside it, a decoded one inside its+ # encoded spelling — and a single raw occurrence must count once.+ # Two distinct links cannot share bytes, so overlapping spans are+ # always readings of one occurrence and the outermost is the span+ # the constructs anchor from.+ def coalesce(spans)+ kept = []+ spans.each do |span|+ last = kept.last+ next if last && span.offset < last.offset + last.length++ kept << span+ end+ kept+ end++ # Scanned over the whole body once, because a pattern cannot start+ # its search at a byte offset the way `byteindex` does.+ def pattern_spans(reading)+ @pattern_spans[reading.text] ||= begin+ spans = []+ pos = 0+ while (match = reading.pattern.match(@input, pos))+ from, to = match.byteoffset(0)+ unless scheme_tail?(reading, from)+ spans << Occurrence.new(offset: from, length: to - from)+ end+ pos = match.begin(0) + 1+ end+ spans.freeze+ end+ end++ # A pattern that also accepts the raw spellings the engine's href+ # escaped on its way in: markdown-it escapes every unsafe character+ # of a destination and keeps the author's own `%xx` verbatim, so+ # each escape in the href may stand for either, independently.+ # `<…/search?q="%20%40name">` reaches the engine as+ # `…/search?q=%22%20%40name%22`. Nil unless the value has such an+ # escape.+ def url_pattern(text)+ parts = []+ mixed = false+ pos = 0++ while (match = /%\h\h(?:%\h\h)*/.match(text, pos))+ parts << Regexp.escape(text[pos...match.begin(0)])+ escapes = match[0]+ pos = match.end(0)++ decoded = percent_decode(escapes)+ if decoded.nil?+ parts << Regexp.escape(escapes)+ next+ end++ cursor = 0+ decoded.each_char do |char|+ escaped = escapes[cursor, 3 * char.bytesize]+ cursor += escaped.length+ if char.match?(URL_SAFE)+ parts << Regexp.escape(escaped)+ else+ mixed = true+ parts << "(?:#{Regexp.escape(escaped)}|#{literal_branch(char)})"+ end+ end+ end+ return nil unless mixed++ parts << Regexp.escape(text[pos..])+ Regexp.new(parts.join)+ end++ # A raw `%` only reaches the href as `%25` when it starts no escape+ # of its own; everything else stands for itself.+ def literal_branch(char)+ char == "%" ? "%(?!\\h\\h)" : Regexp.escape(char)+ end++ def url_readings(value)+ texts = [value]+ decoded = percent_decode(value)+ texts << decoded if decoded && decoded != value+ schemeful = texts.size++ bare = value.sub(%r{\Ahttps?://}i, "")+ if bare != value+ texts << bare+ bare_decoded = percent_decode(bare)+ texts << bare_decoded if bare_decoded && !texts.include?(bare_decoded)+ end++ texts.each_with_index.map do |text, index|+ Reading.new(text:, schemeless: index >= schemeful, pattern: url_pattern(text))+ end+ end++ # Percent-decoding only — `CGI.unescape` would also turn `+` into a+ # space, which is form encoding. Decoding runs on the bytes: a value+ # mixing non-ASCII characters with `%xx` would otherwise raise on+ # substituting a binary escape into a UTF-8 string.+ def percent_decode(value)+ return value unless value.include?("%")++ decoded =+ value+ .b+ .gsub(/%\h\h/) { |encoded| [encoded[1, 2]].pack("H2") }+ .force_encoding(Encoding::UTF_8)+ decoded.valid_encoding? ? decoded : nil+ end++ def construct_match_at(anchor, occurrence, allow_prefix: false)+ byte = @input.getbyte(anchor)+ occurrence_end = occurrence.offset + occurrence.length+ @scanner.url_dispatch[byte]&.each do |construct|+ match = construct.detect(@input, anchor, byte)+ next if match.nil?+ next unless match.start_pos <= occurrence.offset+ if match.end_pos >= occurrence_end+ return match+ elsif allow_prefix && match.end_pos > occurrence.offset &&+ swallowed_tail?(match.end_pos, occurrence_end)+ return match+ end+ end+ nil+ end++ # See {Constructs::Base.swallowed_tail?}.+ def swallowed_tail?(from, to)+ Constructs::Base.swallowed_tail?(@input.byteslice(from, to - from))+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/substitution_pass.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/substitution_pass.rbnew file mode 100644index 00000000000..cf2f07fb48b--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/substitution_pass.rb@@ -0,0 +1,359 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ class EngineScanner+ # The escalation when count matching refused a body: confirm each+ # candidate occurrence on its own. One occurrence's bytes are replaced+ # with a marker word and the body is parsed again. When exactly one+ # instance of that construct disappears from the parse and nothing+ # appears that the marker itself does not explain, then the+ # occurrence was that construct, at that position. The check compares+ # token multisets and needs no line maps, so bodies with CR line+ # endings or entity spellings can use it too.+ #+ # An occurrence the engine skips changes nothing when replaced: its+ # delta is empty and it stays unconfirmed, which leaves it unchanged+ # (see {MarkdownScanner}). The confirmed ones are still extracted.+ #+ # Each candidate occurrence costs one engine parse, limited by+ # {MAX_SUBSTITUTIONS}. Count matching refuses well under 1% of+ # engine-tier bodies, so this stays cheap.+ class SubstitutionPass+ # A body with hundreds of occurrences of a tracked value keeps its+ # tail unconfirmed instead of paying hundreds of parses. One check+ # costs about a millisecond; a corpus run needed fewer than ninety+ # for every body but a generated keyword report.+ MAX_SUBSTITUTIONS = 128++ # Default time limit across a body's substitution checks. The parse+ # count alone does not bound the work: one generated worst-case body+ # can push every parse toward the context timeout, so the wall clock+ # stops first. A scanner on its slow retry passes a larger budget,+ # because there a single legitimate parse can take longer than this+ # whole default.+ SUBSTITUTION_SECONDS_BUDGET = 10.0++ def initialize(+ scanner,+ input,+ data,+ locator,+ cause,+ seconds_budget: SUBSTITUTION_SECONDS_BUDGET+ )+ @scanner = scanner+ @input = input+ @data = data+ @locator = locator+ @cause = cause+ @seconds_budget = seconds_budget+ @substitutions = 0+ @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)+ @limit_hit = nil+ @unplaced_cause = nil+ end++ def result+ base = construct_multiset(@data)+ expected = tracked_expected(base)+ marker = build_marker++ # A body with CR line endings skips the count-matching pass, so+ # `EngineScanner::MAX_SCANNED_VALUES` is checked here too.+ url_values = expected.count { |entry| entry[:kind] == :url }+ if url_values > MAX_SCANNED_VALUES+ return Result.new(output: @input, cause: :url_volume)+ end+ if expected.size - url_values > MAX_SCANNED_VALUES+ return Result.new(output: @input, cause: :name_volume)+ end++ # Once the limit or the budget is hit the loops stop, and the tail+ # then counts as unconfirmed through the expected totals below.+ spans = {}+ confirmed = 0+ expected.each do |entry|+ break if @limit_hit+ occurrences_for(entry).each do |occurrence|+ break if @limit_hit+ next unless confirm(base, entry, occurrence, marker) == :confirmed+ confirmed += 1 if place(entry, occurrence, spans)+ end+ end+ unconfirmed_quotes = confirm_quotes(base, marker, spans)++ ordered, dropped = without_overlaps(spans.values.sort_by(&:start_pos))+ confirmed -= dropped+ unconfirmed = expected.sum { |entry| entry[:count] } - confirmed + unconfirmed_quotes++ Result.new(+ output: ordered.empty? ? @input : @locator.splice(ordered),+ cause: unconfirmed > 0 ? (@limit_hit || @unplaced_cause || @cause) : nil,+ )+ end++ private++ # Every construct value, the inline-code span count, and the+ # block-token inventory. Line maps are left out on purpose; they are+ # what a CR body cannot provide.+ def construct_multiset(data)+ counts = Hash.new(0)++ data["blocks"].each do |block|+ block["mentions"].each { |content| counts[[:mention, content]] += 1 }+ block["hashtags"].each { |hashtag| counts[[:hashtag, hashtag["slug"]]] += 1 }+ block["links"].each { |link| counts[[:url, link["href"]]] += 1 }+ block["images"].each { |src| counts[[:url, src]] += 1 }+ block["emojis"].each { |name| counts[[:emoji, name]] += 1 }+ counts[:code] += block["code"]+ end+ data["blockTokens"].each do |token|+ key = [:block, token["type"], token["tag"]]+ key.push(token["username"], token["post"], token["topic"]) if quote_key?(key)+ counts[key] += 1+ end++ counts+ end++ # The denominator for `unconfirmed`. Quotes are handled separately:+ # a quote header may carry nothing to remap, so its extraction is+ # best-effort.+ def tracked_expected(base)+ base.filter_map do |key, count|+ next unless key.is_a?(Array)++ kind, value = key+ # The multiset key keeps the engine's exact value so a+ # substitution delta still matches its token; the raw text to+ # look for is folded.+ text =+ case kind+ when :mention+ FoldedText.fold(value) if @scanner.mention_tracked?(value.delete_prefix("@"))+ when :hashtag+ hashtag = @scanner.hashtag_text(value)+ FoldedText.fold(hashtag) if hashtag+ when :emoji+ FoldedText.fold(":#{value}:") if @scanner.emoji_tracked?(value)+ when :url+ value if @scanner.url_tracked?(value)+ end+ next if text.nil?++ { kind:, key:, text:, count: }+ end+ end++ def occurrences_for(entry)+ if entry[:kind] == :url+ @locator.url_spans(entry[:text], [email protected])+ else+ @locator.folded_occurrences(entry[:kind], entry[:text])+ end+ end++ # The delta rule: replacing the occurrence must remove exactly one+ # instance of the target construct, and may only add constructs that+ # spell the marker word. Anything else fails the check: a block+ # appearing or vanishing, a suppressed construct showing up, the+ # code-span count changing.+ #+ # The outcome says why a check did not confirm: `:not_construct`+ # (empty delta — the occurrence is not a live construct here, so+ # skipping it is correct), `:mismatch`, `:limit`/`:budget` when no+ # check ran. The distinction lets the caller count real unconfirmed+ # constructs without counting shielded look-alikes.+ def confirm(base, entry, occurrence, marker)+ elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started_at+ if elapsed > @seconds_budget+ @limit_hit = :substitution_budget+ return :budget+ end+ if (@substitutions += 1) > MAX_SUBSTITUTIONS+ @limit_hit ||= :substitution_limit+ return :limit+ end++ substituted_body =+ "#{@input.byteslice(0, occurrence.offset)}#{replacement_for(entry, marker)}" \+ "#{@input.byteslice((occurrence.offset + occurrence.length)..)}"+ begin+ substituted =+ construct_multiset(+ @scanner.engine_scan([{ id: nil, raw: substituted_body }]).first,+ )+ rescue EngineScanner::RetryDeadlineError+ @limit_hit = :substitution_budget+ return :budget+ rescue MiniRacer::ScriptTerminatedError+ # On the slow retry the call's ceiling is the time left until+ # the per-body deadline, so a terminated check means the+ # deadline is spent, not that the engine failed.+ raise unless @scanner.retry_deadline_active?++ @scanner.reset_engine!+ @limit_hit = :substitution_budget+ return :budget+ end++ removed = diff(base, substituted)+ added = diff(substituted, base)+ return :not_construct if removed.empty? && added.empty?++ if entry[:kind] == :quote+ # An opener core made no quote of (no closer, say) is text; the+ # header may hold a mention that moves, but no quote does.+ return :not_construct if removed.keys.none? { |key| quote_key?(key) }++ # The marker keeps the quote a quote, so only its header fields+ # move; the body's own blocks stay where they are.+ unless removed.size == 1 && quote_key?(removed.keys.first) &&+ removed.values.first == 1+ return :mismatch+ end+ else+ return :mismatch if removed.keys != [entry[:key]]+ return :mismatch if removed[entry[:key]] != 1+ end++ added.each_key do |key|+ unless key.is_a?(Array) && key.any? { |part| part.to_s.include?(marker) }+ return :mismatch+ end+ end+ :confirmed+ end++ def diff(left, right)+ left.each_with_object({}) do |(key, count), result|+ difference = count - right.fetch(key, 0)+ result[key] = difference if difference > 0+ end+ end++ def place(entry, occurrence, spans)+ if entry[:kind] == :url+ match =+ @locator.anchor_match(occurrence) ||+ @locator.bare_value_match(entry[:key][1], occurrence)++ # The position is confirmed, but no grammar can take the+ # construct whole (an escaped-bracket label, a form beyond the+ # pattern caps). It stays unchanged AND counts as unconfirmed —+ # a stale reference the caller must hear about.+ if match.nil?+ @unplaced_cause ||= @scanner.unplaced_url_cause(entry[:key][1])+ return false+ end+ else+ match = @locator.node_match(entry[:kind], occurrence)+ end++ # Two readings of one URL, or a self-link's label and destination,+ # can both confirm the same match.+ key = [match.start_pos, match.end_pos]+ return false if spans.key?(key)++ spans[key] = match+ true+ end++ # Quote openers: the target delta is one `blockquote` bbcode block+ # disappearing. Returns how many openers stay unconfirmed but+ # remappable. Two cases do not count: a header that parses but has+ # no username (nothing to remap), and a check that answered "not a+ # live construct" (a `[quote=` inside a code fence).+ def confirm_quotes(base, marker, spans)+ return 0 if base.keys.none? { |key| quote_key?(key) }++ quote_entry = { kind: :quote, key: nil, text: nil }+ unconfirmed = 0+ pos = 0+ while (offset = @input.byteindex(/\[quote=/i, pos))+ # Once the limit or the budget is hit the body carries its+ # refusal anyway, so scanning the remaining openers would only+ # cost time.+ break if @limit_hit+ pos = offset + 1+ match = @scanner.quote_construct.detect_block_opener(@input, offset)+ if match.nil?+ unconfirmed += 1 unless @scanner.quote_construct.parseable_opener?(@input, offset)+ next+ end++ occurrence =+ Locator::Occurrence.new(+ offset: match.start_pos,+ length: match.end_pos - match.start_pos,+ )+ outcome = confirm(base, quote_entry, occurrence, marker)+ if outcome == :confirmed+ spans[[match.start_pos, match.end_pos]] ||= match+ elsif outcome != :not_construct+ unconfirmed += 1+ end+ end++ unconfirmed+ end++ # The later one is counted back as unconfirmed, so the body keeps+ # its refusal cause.+ def without_overlaps(ordered)+ kept = []+ dropped = 0++ ordered.each do |match|+ if kept.empty? || match.start_pos >= kept.last.end_pos+ kept << match+ else+ dropped += 1+ end+ end++ [kept, dropped]+ end++ # The marker keeps the shape core dispatches on, or the marked parse+ # gains or loses tokens the occurrence never produced: an+ # `upload://` source keeps its scheme (core leaves an upload+ # image's alt alone but linkifies a URL in any other alt), a web+ # URL keeps its scheme, and a quote opener stays a quote opener.+ def replacement_for(entry, marker)+ return "[quote=\"#{marker}\"]" if entry[:kind] == :quote+ return marker unless entry[:kind] == :url++ value = entry[:key][1]+ if value.start_with?("upload://")+ "upload://#{marker}#{File.extname(value)}"+ elsif (scheme = value[%r{\A(https?)://}i, 1])+ # An autolink `<…>` stays an autolink only with a scheme; the+ # bare word alone on a line would parse as an HTML block.+ "#{scheme}://#{marker}"+ else+ marker+ end+ end++ def quote_key?(key)+ key.is_a?(Array) && key[0] == :block && key[1] == "bbcode_open" &&+ key[2] == "blockquote"+ end++ def build_marker+ counter = 0+ marker = "substmark0"+ marker = "substmark#{counter += 1}" while @input.include?(marker)+ marker+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/folded_text.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/folded_text.rbnew file mode 100644index 00000000000..84cc6aa32fd--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/folded_text.rb@@ -0,0 +1,91 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ # One folded copy of a post body, with the byte map back to the raw.+ #+ # The engine reports a folded spelling for a name: it lowercases an+ # emoji shortcode and a hashtag slug before it looks either up. Both+ # sides therefore go through {.fold} — a fold at least as coarse as the+ # engine's is what count matching needs (see {MarkdownScanner}).+ #+ # The map back is per grapheme cluster: composition and case mapping+ # both stay inside one cluster, so a cluster's folded bytes belong to+ # exactly that cluster's raw bytes. A folded hit that does not start and+ # end on cluster boundaries denotes no raw span and is dropped, which+ # can only lower the count and so only ever escalates.+ class FoldedText+ # @param text [String]+ # @return [String] the folded spelling+ def self.fold(text)+ # NFC leaves ASCII alone, and most bodies are ASCII.+ if text.ascii_only?+ text.downcase+ else+ Migrations::NameNormalizer.normalize(text).tr("ς", "σ")+ end+ end++ # @param raw [String] the post body+ def initialize(raw)+ @raw = raw+ if raw.ascii_only?+ @text = raw.downcase+ else+ build_map+ end+ end++ # @return [String] the whole body, folded+ attr_reader :text++ # @param offset [Integer] byte offset into {#text}+ # @param length [Integer] byte length within {#text}+ # @return [Array(Integer, Integer), nil] the raw offset and length, or+ # nil when the folded span crosses a cluster boundary+ def raw_span(offset, length)+ return offset, length if @cluster_starts.nil?+ return nil unless cluster_start?(offset)++ last = offset + length - 1+ return nil unless offset + length == @text.bytesize || cluster_start?(offset + length)++ start = @cluster_starts[offset]+ [start, @cluster_starts[last] + @cluster_lengths[last] - start]+ end++ private++ # Cluster raw offsets rise strictly, so a change in the recorded raw+ # offset marks a cluster boundary.+ def cluster_start?(offset)+ offset.zero? || @cluster_starts[offset] != @cluster_starts[offset - 1]+ end++ # Records, for every folded byte, where its cluster starts in the raw+ # and how long it is there. A cluster whose fold is empty contributes+ # no folded byte, so no occurrence can span its raw bytes.+ def build_map+ @text = +""+ @cluster_starts = []+ @cluster_lengths = []+ offset = 0++ @raw.each_grapheme_cluster do |cluster|+ length = cluster.bytesize+ folded = FoldedText.fold(cluster)+ folded.bytesize.times do+ @cluster_starts << offset+ @cluster_lengths << length+ end+ @text << folded+ offset += length+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/hashtag_reference.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/hashtag_reference.rbnew file mode 100644index 00000000000..218d4eda66e--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/hashtag_reference.rb@@ -0,0 +1,18 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ # A deferred hashtag (`#slug`, `#parent:child`, or a forced `#name::tag`+ # / `#name::category`). `name` is the text between the `#` and any `::`+ # suffix, so it keeps the `parent:child` separator but drops the suffix.+ # `forced_type` is `:category`, `:tag`, or nil when the source didn't+ # pin the type; the importer classifies the untyped ones. The category+ # or tag is named, not identified — its slug or name can change at+ # import.+ HashtagReference = Data.define(:name, :forced_type)+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/internal_link_reference.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/internal_link_reference.rbnew file mode 100644index 00000000000..4b2b408de6c--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/internal_link_reference.rb@@ -0,0 +1,56 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ # A deferred internal link — a URL pointing at another record on the+ # same Discourse. The entity it names is resolved at import (ids and+ # slugs change), so the node carries what the URL revealed.+ #+ # `url` is the full original URL and the importer's fallback. `text` is+ # a markdown link's link text, or nil for a bare URL (which the renderer+ # emits bare, so oneboxes keep working). `target_type` is a symbol+ # naming the kind (`:topic`, `:post`, `:user`, `:category`, `:tag`,+ # `:group`, `:badge`, or `:site`); the {RawExtractor} maps it to the+ # stored enum.+ #+ # For a routed target exactly one addressing form is filled: `target_id`+ # (a source `original_id`), `target_name` (a username, group/tag name,+ # or a `parent:child` category slug path), or `target_topic_id` ++ # `target_post_number`. `target_suffix` is whatever trailed the matched+ # route, reattached verbatim at render.+ #+ # The multi-coordinate tag routes carry a second coordinate in+ # `target_tag_path`: a `:category_tag` link names its category through+ # `target_id`/`target_name` as above, plus the tag as+ # `[none/|all/]<tag-name>`; a `:tag_intersection` link names only tags,+ # `<t1>/<t2>[/…]`. Tag names come from URL path segments, so `/` can't+ # occur inside one.+ #+ # A `:site` target names no record: no addressing field is filled,+ # `target_suffix` carries the whole path/query/fragment, and the+ # importer renders the destination base URL plus that suffix.+ #+ # `url_offset` is the destination's byte offset within the verbatim+ # source snippet, and `label_url_offset` the offset of a self-link label+ # spelling the same destination (nil otherwise); both spans are+ # `url.bytesize` long.+ InternalLinkReference =+ Data.define(+ :url,+ :text,+ :target_type,+ :target_id,+ :target_name,+ :target_topic_id,+ :target_post_number,+ :target_tag_path,+ :target_suffix,+ :url_offset,+ :label_url_offset,+ )+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/quote_reference.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/quote_reference.rbnew file mode 100644index 00000000000..96e54a94560--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/quote_reference.rb@@ -0,0 +1,21 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ # A deferred quote reference: the opening `[quote="…"]` tag alone. That+ # tag is all the construct consumes and all that carries references+ # needing remapping — the body and `[/quote]` stay in the raw and are+ # scanned like any other text.+ #+ # `name` is the quoted user's display name, kept only when the header+ # gives one that differs from the username; it's the fallback text when+ # the user can't be mapped at import. `post_number` and `topic_id` are+ # the source's own coordinates of the quoted post, not Discourse ids;+ # the importer turns them into a source post `original_id`.+ QuoteReference = Data.define(:username, :name, :post_number, :topic_id)+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/tier_gate.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/tier_gate.rbnew file mode 100644index 00000000000..963f4186032--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/tier_gate.rb@@ -0,0 +1,340 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ # Classifies a post body before any extraction runs (measured on a real+ # corpus: well over half of all posts leave through `:none`):+ #+ # :none - nothing extractable; the body is returned untouched.+ # :engine - at least one candidate construct; the {EngineScanner}+ # extracts it against the real parse.+ #+ # Every check errs toward `:engine`, so a wrong guess costs an engine+ # parse, never a wrong extraction.+ class TierGate+ # A body with none of these characters cannot hold any built-in+ # construct: `@` (mention), `[` (quote/attachment/image), `#`+ # (hashtag), the `uploads/` segment of a full upload URL or the+ # `original/`/`optimized/` storage segment of an S3/CDN one. Named+ # character entities get their own alternative because `@bob`+ # spells a construct while containing none of the trigger characters;+ # numeric forms all contain `#`.+ BASE_PRESENCE = %r{[@\[#]|uploads/|(?:original|optimized)/|&[a-zA-Z][a-zA-Z0-9]{1,31};}+ private_constant :BASE_PRESENCE++ # Numeric character references, decoded and tested below: `@bob`+ # can spell a mention with no literal `@` in the body.+ NUMERIC_ENTITY = /&#(x\h{1,7}|\d{1,7});/i+ private_constant :NUMERIC_ENTITY++ NAMED_ENTITY = /&([a-zA-Z][a-zA-Z0-9]{1,31});/+ private_constant :NAMED_ENTITY++ # A character that could take part in a construct or its name once an+ # entity decodes to it: Unicode alphanumerics and marks (unicode+ # usernames), the name interior characters, the trigger characters,+ # and the URL characters a route needs.+ CONSTRUCT_CHAR = %r{[\p{Alnum}\p{M}_@#:./-]}+ private_constant :CONSTRUCT_CHAR++ # Named entities that decode to a character no construct can contain,+ # so that an ordinary pasted `&` or `…` makes no candidate+ # here. markdown-it ships its full name table only as an encoded trie,+ # hence this explicit allowlist; a spec decodes each name through the+ # real engine and fails if one can form a construct character.+ # Anything not listed counts as construct-capable — an unknown name+ # costs time, not correctness.+ IRRELEVANT_NAMED_ENTITIES = %w[+ amp+ AMP+ apos+ asymp+ bdquo+ bull+ cent+ copy+ curren+ dagger+ Dagger+ darr+ deg+ divide+ ensp+ emsp+ euro+ ge+ gt+ GT+ harr+ hellip+ infin+ laquo+ larr+ ldquo+ le+ lrm+ lsquo+ lt+ LT+ mdash+ middot+ nbsp+ ndash+ ne+ para+ plusmn+ pound+ prime+ Prime+ quot+ QUOT+ raquo+ rarr+ rdquo+ reg+ rlm+ rsquo+ sbquo+ sect+ shy+ thinsp+ times+ trade+ uarr+ yen+ zwj+ zwnj+ ].to_set.freeze++ # A name-gated construct's probe: which bytes open it, the folded+ # names it accepts, and the byte that has to close the name (only the+ # emoji shortcode has one).+ Probe =+ Data.define(:names, :triggers, :terminator) do+ def initialize(names:, triggers:, terminator: nil)+ super+ end+ end+ private_constant :Probe++ # How many raw bytes a name of N folded bytes may span. Folding maps+ # each grapheme cluster on its own and can shrink one — a decomposed+ # cluster composing into a single character, `K` (U+212A) downcasing+ # to `k` — so a raw spelling can be longer than the name it folds to.+ NAME_WINDOW_FACTOR = 4+ private_constant :NAME_WINDOW_FACTOR++ # @param constructs [Array<Constructs::Base>] the same list the+ # {EngineScanner} anchors with, so the gate follows what is wired.+ def initialize(constructs:)+ @presence = Regexp.union(BASE_PRESENCE, *constructs.filter_map(&:presence_pattern))++ # The name-gated constructs answer "could anything be extracted?" by+ # name: a probe folds the bytes after each trigger and asks their+ # name sets. Every other construct extracts wherever its pattern+ # matches, so presence alone makes a body worth scanning.+ probed = []+ @unconditional_patterns = []+ unknown = false++ constructs.each do |construct|+ case construct+ when Constructs::Mention, Constructs::Hashtag+ probed << Probe.new(names: construct.names, triggers: construct.triggers)+ when Constructs::Emoji+ # Requiring the closing `:` keeps every colon a tracked name+ # happens to follow from becoming a candidate — a colon is far+ # more common in prose than `@` or `#`.+ probed << Probe.new(+ names: construct.names,+ triggers: construct.triggers,+ terminator: ":".ord,+ )+ when Constructs::Quote, Constructs::Upload, Constructs::UploadUrl+ # Covered by the fixed checks in `unconditional_candidate?`.+ when Constructs::InternalLink+ @unconditional_patterns << construct.presence_pattern+ else+ # An unknown construct cannot be ruled out, so candidacy is+ # assumed whenever presence hits.+ unknown = true+ end+ end++ @assume_candidates = unknown++ @probe_dispatch = {}+ probed.each do |probe|+ probe.triggers.each { |char| (@probe_dispatch[char.ord] ||= []) << probe }+ end+ @probe_dispatch.each_value(&:freeze)+ @probe_dispatch.freeze++ @probe_stop =+ if probed.empty?+ nil+ else+ chars = probed.flat_map(&:triggers).uniq+ Regexp.new("[#{chars.map { |char| Regexp.escape(char) }.join}]")+ end+ end++ # @param raw [String]+ # @return [Symbol] `:none` or `:engine`+ def classify(raw)+ return :none unless raw.match?(@presence)++ # A construct-capable entity is a candidate no byte check can see.+ return :engine if TierGate.construct_capable_entity?(raw)++ real_candidates?(raw) ? :engine : :none+ end++ # Whether `text` holds a character reference that can decode into a+ # construct-relevant character. The gate and the engine tier share+ # this blind spot, so both read it here.+ def self.construct_capable_entity?(text)+ return false unless text.include?("&")++ text.scan(NUMERIC_ENTITY) do+ code = Regexp.last_match(1)+ return true if construct_codepoint?(code)+ end++ text.scan(NAMED_ENTITY) do+ return true unless IRRELEVANT_NAMED_ENTITIES.include?(Regexp.last_match(1))+ end++ false+ end++ # The byte offsets of every construct-capable reference in `text`,+ # sorted, so the engine tier checks a region with a binary search+ # instead of slicing and rescanning it per region.+ def self.construct_capable_entity_offsets(text)+ return [] unless text.include?("&")++ offsets = []+ [+ [NUMERIC_ENTITY, ->(match) { construct_codepoint?(match[1]) }],+ [NAMED_ENTITY, ->(match) { !IRRELEVANT_NAMED_ENTITIES.include?(match[1]) }],+ ].each do |pattern, capable|+ pos = 0+ while (match = pattern.match(text, pos))+ offsets << match.byteoffset(0).first if capable.call(match)+ pos = match.end(0)+ end+ end+ offsets.sort!+ offsets+ end++ private++ def real_candidates?(raw)+ @assume_candidates || unconditional_candidate?(raw) || probe_candidate?(raw)+ end++ # The `=` after `[quote` is required: only an opener with metadata+ # holds user/post/topic fields to remap, and both passes search+ # openers with exactly this shape. The `upload://` scheme stays+ # case-sensitive because core's upload rewriting is; an `UPLOAD://`+ # link carries nothing an importer could resolve. Full upload URLs go+ # through the construct's own check, so an unrelated `/uploads/` path+ # makes no candidate.+ def unconditional_candidate?(raw)+ raw.match?(/\[quote=/i) || raw.include?("upload://") ||+ Constructs::UploadUrl.candidate?(raw) ||+ @unconditional_patterns.any? { |pattern| raw.match?(pattern) }+ end++ # Asks at each trigger position whether the bytes after it could spell+ # a tracked name — a superset of what the engine can recognize there,+ # with no boundary or name grammar, so `\@bob` (which core cooks) is a+ # candidate like `@bob` is. Over-approximating costs an engine parse;+ # ruling out wrongly costs a construct.+ def probe_candidate?(raw)+ return false unless @probe_stop++ pos = 0+ while (index = raw.byteindex(@probe_stop, pos))+ @probe_dispatch[raw.getbyte(index)]&.each do |probe|+ return true if tracked_prefix?(raw, index + 1, probe)+ end+ pos = index + 1+ end+ false+ end++ # Whether some prefix of the text after a trigger folds to one of the+ # construct's names. An ASCII window folds by downcasing it whole; a+ # multibyte one grows its prefixes by grapheme cluster, preserving+ # the distinctions in the source's identity keys.+ def tracked_prefix?(raw, from, probe)+ max_bytes = probe.names.max_byte_length+ return false if max_bytes == 0++ window = probe_window(raw, from, max_bytes)+ return false if window.empty?++ if window.ascii_only?+ ascii_tracked_prefix?(raw, from, probe, window.downcase, max_bytes)+ else+ folded_tracked_prefix?(raw, from, probe, window, max_bytes)+ end+ end++ def ascii_tracked_prefix?(raw, from, probe, folded, max_bytes)+ length = 0+ limit = [max_bytes, folded.bytesize].min+ while (length += 1) <= limit+ next unless probe.names.include?(folded.byteslice(0, length))+ return true if probe.terminator.nil? || raw.getbyte(from + length) == probe.terminator+ end+ false+ end++ def folded_tracked_prefix?(raw, from, probe, window, max_bytes)+ folded = +""+ offset = from+ window.each_grapheme_cluster do |cluster|+ folded << Migrations::NameNormalizer.normalize(cluster)+ offset += cluster.bytesize+ break if folded.bytesize > max_bytes+ next unless probe.names.include?(folded)+ return true if probe.terminator.nil? || raw.getbyte(offset) == probe.terminator+ end+ false+ end++ # The bytes a name may reach over: never across whitespace, since no+ # value the engine reports for these constructs holds any, and never+ # more than {NAME_WINDOW_FACTOR} times the longest name.+ def probe_window(raw, from, max_bytes)+ limit = [from + max_bytes * NAME_WINDOW_FACTOR, raw.bytesize].min+ stop = from+ while stop < limit+ byte = raw.getbyte(stop)+ break if byte == 0x20 || (byte >= 0x09 && byte <= 0x0d)+ stop += 1+ end+ # A window cut at the byte limit must not cut a character in half.+ stop += 1 while stop < raw.bytesize && (raw.getbyte(stop) & 0xC0) == 0x80+ raw.byteslice(from, stop - from)+ end++ def self.construct_codepoint?(code)+ codepoint = code.start_with?("x", "X") ? code[1..].to_i(16) : code.to_i+ return false if codepoint == 0 || codepoint > 0x10ffff+ return false if codepoint >= 0xd800 && codepoint <= 0xdfff++ CONSTRUCT_CHAR.match?(codepoint.chr(Encoding::UTF_8))+ end+ private_class_method :construct_codepoint?+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/upload_url_reference.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/upload_url_reference.rbnew file mode 100644index 00000000000..e2e998d87ce--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/upload_url_reference.rb@@ -0,0 +1,23 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ # An upload referenced by a full URL (`/uploads/…`, `/secure-uploads/…`,+ # or an absolute form on any host) rather than a short `upload://` URL.+ # Older posts store these, and the file is lost with the old site unless+ # we carry it over.+ #+ # `sha1` is the upload's 40-hex sha1: read from the basename for a long+ # URL, decoded from the base62 token for a `/uploads/short-url/…` one.+ # `host` is the URL's host as {UrlOrigin} reads origins, nil for a+ # relative URL. `rest` is the URL's path part. The extractor uses host+ # and path to mark rows the importer maps only against an explicit+ # allowlist, so a foreign-looking sha1 cannot rewrite another site's+ # file on a hash collision (see `RawExtractor#upload_ownership`).+ UploadUrlReference = Data.define(:sha1, :host, :rest)+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/url_origin.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/url_origin.rbnew file mode 100644index 00000000000..6ace518f8b9--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/url_origin.rb@@ -0,0 +1,107 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ module MarkdownScanner+ # The one URL-origin reading shared by everything that must agree on+ # what "the source's own URL" means: the {Constructs::InternalLink}+ # grammar, the {EngineScanner}'s tracked-value filter, and the+ # foreign-host signal.+ module UrlOrigin+ # Splits into scheme, `//host` and the rest (path/query/fragment). The+ # host stops at `?`/`#`, so `https://host?ref=x` is the site root with+ # a query, not a host with a `?` in its name. Anything the pattern+ # can't take whole (`mailto:…`, a bare word) isn't an internal URL.+ SPLIT = %r{\A(?:(?:(?<scheme>(?i:https?)):)?//(?<host>[^/?\#]+))?(?<rest>[/?\#]\S*)?\z}+ private_constant :SPLIT++ # Keeps `/forum` from matching inside `/forumxyz`.+ PREFIX_BOUNDARY = %r{\A[/?#]}+ private_constant :PREFIX_BOUNDARY++ # Only a scheme's OWN default port is dropped (`:80` for http, `:443`+ # for https): `https://host:80` is a non-default origin, and a+ # protocol-relative `//host:8443` has no scheme to default from — both+ # keep their port as part of the host identity and match only a+ # configured host spelled with it.+ #+ # @return [Array(String, String), nil] `[host, rest]` for an+ # internal-URL shape, else nil. `host` is nil for a relative URL.+ def self.split(url)+ match = SPLIT.match(url)+ return nil unless match++ host = match[:host]&.downcase+ if host+ case match[:scheme]&.downcase+ when "http"+ host = host.delete_suffix(":80")+ when "https"+ host = host.delete_suffix(":443")+ end+ end++ rest = match[:rest]+ # A host with no path is the site's front page, whose origin needs+ # rewriting like any other. Without a host there is nothing to+ # rewrite.+ return nil if host.nil? && rest.nil?++ [host, rest || ""]+ end++ # Where a URL sits relative to the source's own hosts and prefixes.+ # `path` is the remainder inside the owning prefix when the URL is the+ # source's own, else nil. `prefix` is what `path` was measured+ # against, so a caller that resolves the host from one spelling of a+ # URL can measure a second spelling against the same prefix. `foreign`+ # marks an absolute URL whose host is not configured.+ Origin = Data.define(:host, :rest, :path, :prefix, :foreign)++ # Combines {.split} and {.path_within_prefix} with the configured+ # hosts. Returns nil for anything that is not an internal-URL shape.+ #+ # @param hosts [Hash{String => (String, nil)}] host => path prefix+ # @param base_prefix [String, nil] the prefix for relative URLs+ # @return [Origin, nil]+ def self.classify(url, hosts:, base_prefix:)+ host, rest = split(url)+ return nil if rest.nil?++ if host+ unless hosts.key?(host)+ return Origin.new(host:, rest:, path: nil, prefix: nil, foreign: true)+ end++ prefix = hosts[host]+ else+ prefix = base_prefix+ end++ Origin.new(+ host:,+ rest:,+ path: path_within_prefix(rest, prefix),+ prefix:,+ foreign: false,+ )+ end++ # The path remainder after `prefix`, or nil when the URL falls outside+ # it (it belongs to another app beside the forum). A nil prefix (root+ # install) owns every path. The prefix must end on a real segment+ # boundary: `/forum/t/…` and `/forum` match, `/forumxyz` does not.+ def self.path_within_prefix(rest, prefix)+ return rest if prefix.nil?+ return "" if rest == prefix+ return nil unless rest.start_with?(prefix)++ remainder = rest[prefix.length..]+ PREFIX_BOUNDARY.match?(remainder) ? remainder : nil+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/mention_classifier.rb b/migrations/converters/lib/migrations/converters/discourse/mention_classifier.rbnew file mode 100644index 00000000000..6039613aeb7--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/mention_classifier.rb@@ -0,0 +1,45 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ # Classifies an `@name` mention into the `mention_type` stored on+ # `embed_mentions`: a `MentionType` enum value for `here`, `all`, `group`+ # or `user`.+ #+ # * `@here` is recognized by the source's `here_mention` site setting+ # — its name is configurable, so we don't hard-code `"here"`.+ # * `@all` is recognized by the literal name `all`.+ # * group mentions are recognized from the source's group names.+ # * everything else is a user mention.+ #+ # Names are {NameNormalizer}-folded, so a mention and a group or user name+ # compare equal however the source encoded them.+ class MentionClassifier+ MentionType = Migrations::Database::IntermediateDB::Enums::MentionType+ private_constant :MentionType++ # @param here_mention [String, nil] the source's `here_mention` setting+ # value. Nil or blank disables here-detection, so no name will+ # classify as HERE.+ # @param group_names [Enumerable<String>] the source's group names.+ def initialize(here_mention: "here", group_names: [])+ @here_mention = NameNormalizer.normalize(here_mention) if here_mention.present?+ @group_names = group_names.map { |name| NameNormalizer.normalize(name) }.to_set+ end++ # @param name [String] the mention name (without the leading `@`).+ # @return [Integer] a {MentionType} enum value.+ def call(name)+ normalized = NameNormalizer.normalize(name)++ return MentionType::HERE if @here_mention && normalized == @here_mention+ return MentionType::ALL if normalized == "all"+ return MentionType::GROUP if @group_names.include?(normalized)++ MentionType::USER+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/raw_extractor.rb b/migrations/converters/lib/migrations/converters/discourse/raw_extractor.rbnew file mode 100644index 00000000000..7b27d3ddbc9--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/raw_extractor.rb@@ -0,0 +1,405 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ # Extracts deferred embeds from a Discourse post's Markdown `raw` and+ # replaces each with a placeholder token, recording a typed descriptor on+ # the embed collector given at construction (an {EmbedBuffer}). The+ # importer's `PlaceholderResolver` rewrites the tokens once the+ # `original_id -> discourse_id` maps exist.+ #+ # The hard part is *not* extracting from places that only look like embeds+ # — inside fenced/indented/inline code, or in a link's label or+ # destination. A {MarkdownScanner::TierGate} classifies every body first:+ # most have nothing extractable and pass through untouched, the rest go to+ # the {MarkdownScanner::EngineScanner} (see {MarkdownScanner}).+ #+ # Polls and events are self-contained (no id remapping needed), so they're+ # left in `raw` verbatim.+ class RawExtractor+ # A body normalized and classified once, ready for bounded batch+ # scanning and later extraction with the matching scan data.+ PreparedBody =+ Data.define(:id, :raw, :topic_id, :tier) do+ def engine_bound?+ tier == :engine+ end+ end++ # The measured throughput sweet spot is 32 ordinary posts per V8 call+ # (see {#scan_batches}). The byte cap keeps those calls from becoming+ # arbitrarily large; a single body over the cap still runs alone, so+ # valid large posts are not rejected for being unbatchable.+ DEFAULT_BATCH_POSTS = 32+ DEFAULT_BATCH_BYTES = 4 * 1024 * 1024++ Constructs = MarkdownScanner::Constructs+ private_constant :Constructs++ HashtagType = Migrations::Database::IntermediateDB::Enums::HashtagType+ private_constant :HashtagType++ LinkTarget = Migrations::Database::IntermediateDB::Enums::LinkTarget+ private_constant :LinkTarget++ # A hashtag node's forced type (from a `::category` / `::tag` suffix)+ # mapped to its stored enum value.+ FORCED_HASHTAG_TYPES = { category: HashtagType::CATEGORY, tag: HashtagType::TAG }.freeze+ private_constant :FORCED_HASHTAG_TYPES++ # A {Constructs::InternalLink} node's target kind mapped to the stored+ # `link_target` enum value.+ LINK_TARGET_TYPES = {+ topic: LinkTarget::TOPIC,+ post: LinkTarget::POST,+ user: LinkTarget::USER,+ category: LinkTarget::CATEGORY,+ tag: LinkTarget::TAG,+ group: LinkTarget::GROUP,+ badge: LinkTarget::BADGE,+ site: LinkTarget::SITE,+ category_tag: LinkTarget::CATEGORY_TAG,+ tag_intersection: LinkTarget::TAG_INTERSECTION,+ }.freeze+ private_constant :LINK_TARGET_TYPES++ # Only a mention, hashtag or custom emoji naming something the source+ # actually has is deferred; anything else stays literal text. That is+ # why the name sets are required: without them every `@word` that parses+ # would be rewritten, including the ones that name nobody. Names arrive+ # {NameNormalizer}-folded (custom emoji names are folded here), and with+ # no custom emoji names the emoji construct is left out entirely, so+ # posts don't pay for its `:` trigger.+ #+ # @param internal_link_hosts [Hash{String => (String, nil)}] the+ # source's own hosts (base URL and former domains), each downcased and+ # mapped to its path prefix — `"/forum"` for a subdirectory install+ # (no trailing slash), `nil` for a root install. Hosts may carry+ # different prefixes. Empty (the default) detects relative links only.+ # @param internal_link_base_prefix [String, nil] the current site's own+ # path prefix, stripped from a relative link before the route is+ # parsed.+ # @param on_foreign_host [#call, nil] called once per host with the host+ # of an absolute, internal-looking link whose host is not in+ # `internal_link_hosts` — a hint that a former domain may be missing+ # from the source_site settings.+ # @param markdown_engine [MarkdownEngine::Context] the engine context+ # for `:engine`-classified bodies. Build it after the worker forks —+ # V8 contexts do not survive forking.+ # @param on_engine_refusal [#call, nil] called with the cause and its+ # diagnostic detail (the exception class name for `:engine_error`,+ # else nil) whenever an `:engine` body keeps at least one unconfirmed+ # construct; also tallied on {#engine_refusals}. The caller knows+ # which post it is extracting, so post identity stays on its side.+ # @param slow_timeout_ms [Integer, nil] the retry ceiling for a body+ # whose parse the engine terminated at the fast default; `nil`+ # disables it.+ # @param on_slow_parse [#call, nil] called with no arguments whenever a+ # body's parse only succeeded on the slow retry; also counted on+ # {#slow_parses}.+ def initialize(+ embeds:,+ mention_names:,+ hashtag_names:,+ markdown_engine:,+ mention_classifier: MentionClassifier.new,+ custom_emoji_names: nil,+ internal_link_hosts: {},+ internal_link_base_prefix: nil,+ on_foreign_host: nil,+ on_engine_refusal: nil,+ slow_timeout_ms: MarkdownScanner::EngineScanner::SLOW_TIMEOUT_MS,+ on_slow_parse: nil+ )+ @embeds = embeds+ @mention_names = mention_names+ @hashtag_names = hashtag_names+ @mention_classifier = mention_classifier+ @markdown_engine = markdown_engine+ @internal_link_hosts = internal_link_hosts+ @internal_link_base_prefix = internal_link_base_prefix+ @on_engine_refusal = on_engine_refusal+ @on_slow_parse = on_slow_parse+ @engine_refusals = Hash.new(0)+ @slow_parses = 0++ constructs = [Constructs::Upload.new, Constructs::UploadUrl.new, Constructs::Quote.new]+ # After UploadUrl, so an upload URL still wins over a bare internal+ # link that happens to look like one.+ constructs << Constructs::InternalLink.new(+ hosts: internal_link_hosts,+ base_prefix: internal_link_base_prefix,+ on_foreign_host:,+ )+ constructs << Constructs::Mention.new(names: mention_names)+ constructs << Constructs::Hashtag.new(names: hashtag_names)+ if custom_emoji_names.present?+ constructs << Constructs::Emoji.new(+ names:+ Migrations::CompactStringSet.new(+ custom_emoji_names.map { |name| Migrations::NameNormalizer.normalize(name) },+ ),+ )+ end++ # The constructs carry no per-post state (the internal-link one only+ # accumulates its report-once host set) and the scanner resets its+ # state on each `scan`, so they are built once and reused. `extract`+ # swaps `@topic_id` per call, so one extractor must not run in two+ # threads at once — each worker holds its own.+ on_node = ->(node, source) { defer(node, source) }+ @gate = MarkdownScanner::TierGate.new(constructs:)+ @engine_scanner =+ MarkdownScanner::EngineScanner.new(+ engine: markdown_engine,+ constructs:,+ internal_link_hosts:,+ internal_link_base_prefix:,+ slow_timeout_ms:,+ &on_node+ )+ end++ # Cause tallies (`cause => count`) for `:engine` bodies that kept at+ # least one unconfirmed construct — the conversion's must-resolve list.+ # Only the first group should read as a problem with the extraction+ # itself:+ #+ # * unexpected failure — `:unanchored` (the engine recognized a+ # tracked occurrence, but no construct grammar could place it),+ # `:engine_error`, `:overlap`;+ # * known unsupported source content — `:invalid_internal_route` (a+ # coordinate-shaped path that parses no route, usually a link+ # already broken on the source), `:count_mismatch`, `:entity`,+ # `:cr_line_endings`;+ # * budget — `:substitution_limit`, `:substitution_budget`,+ # `:url_volume`, `:name_volume`;+ # * the rest of a partially extracted body.+ attr_reader :engine_refusals++ # How many bodies parsed only on the slow retry. They extracted fine, so+ # this is a heads-up count, not part of the must-resolve list.+ attr_reader :slow_parses++ # @param topic_id [Integer, nil] the containing post's source topic id,+ # which completes a quote reference that names a `post:` but no+ # `topic:` (Discourse omits `topic:` within one topic).+ # @return [String, nil] the body with embeds replaced by placeholder+ # tokens. Invalid bytes are scrubbed and non-UTF-8 encodings converted+ # here at the top, exactly once: the gate, the engine and every byte+ # offset must all read the same bytes.+ def extract(raw, topic_id: nil)+ extract_prepared(prepare(raw:, topic_id:))+ end++ # A batching caller prepares each body once, passes the results to+ # {#scan_batches}, then calls {#extract_prepared} with the scan data+ # keyed by id.+ def prepare(raw:, id: nil, topic_id: nil)+ return PreparedBody.new(id:, raw:, topic_id:, tier: :none) if raw.nil?++ normalized = normalize_input(raw)+ PreparedBody.new(id:, raw: normalized, topic_id:, tier: @gate.classify(normalized))+ end++ # Partitions by both count and aggregate bytes. If one partition+ # terminates, only that partition returns no data; its bodies recover+ # through the per-body path in {#extract_prepared}.+ #+ # Batching exists because the per-call V8 overhead (~0.5ms) is+ # comparable to an average parse. On a 1.5M-post corpus at 18 workers,+ # batches of 32 measured 45.2s wall / 360.4s V8 against 70.9s / 684.4s+ # unbatched.+ def scan_batches(+ prepared_bodies,+ max_posts: DEFAULT_BATCH_POSTS,+ max_bytes: DEFAULT_BATCH_BYTES+ )+ validate_batch_limits!(max_posts, max_bytes)+ bodies = prepared_bodies.select(&:engine_bound?)+ ids = bodies.map(&:id)+ raise ArgumentError, "prepared bodies must have non-nil ids" if ids.any?(&:nil?)+ raise ArgumentError, "prepared bodies must have unique ids" if ids.uniq.size != ids.size++ data = {}+ each_scan_batch(bodies, max_posts:, max_bytes:) do |batch|+ data.merge!(scan_prepared_batch(batch))+ end+ data+ end++ def extract_prepared(prepared_body, scan_data: nil)+ @topic_id = prepared_body.topic_id+ return prepared_body.raw unless prepared_body.engine_bound?++ extract_engine(prepared_body.raw, scan_data)+ end++ private++ def validate_batch_limits!(max_posts, max_bytes)+ unless max_posts.is_a?(Integer) && max_posts.positive?+ raise ArgumentError, "max_posts must be a positive integer"+ end+ unless max_bytes.is_a?(Integer) && max_bytes.positive?+ raise ArgumentError, "max_bytes must be a positive integer"+ end+ end++ def each_scan_batch(bodies, max_posts:, max_bytes:)+ batch = []+ bytes = 0++ bodies.each do |body|+ body_bytes = body.raw.bytesize+ if batch.any? && (batch.size >= max_posts || bytes + body_bytes > max_bytes)+ yield batch+ batch = []+ bytes = 0+ end++ batch << body+ bytes += body_bytes+ end++ yield batch if batch.any?+ end++ def scan_prepared_batch(batch)+ posts = batch.map { |body| { id: body.id, raw: body.raw } }+ @markdown_engine.scan(posts).index_by { |data| data["id"] }+ rescue MiniRacer::ScriptTerminatedError, MiniRacer::RuntimeError+ @markdown_engine.reset!+ {}+ end++ def normalize_input(raw)+ if raw.encoding == Encoding::UTF_8+ raw.valid_encoding? ? raw : raw.scrub+ else+ raw.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)+ end+ end++ # The cause is counted and reported so a conversion surfaces how many+ # posts (and why) still need resolution.+ def extract_engine(raw, scan_data)+ result = @engine_scanner.scan(raw, scan_data:)+ if result.slow_parse+ @slow_parses += 1+ @on_slow_parse&.call+ end+ if result.cause+ @engine_refusals[result.cause] += 1+ @on_engine_refusal&.call(result.cause, result.detail)+ end+ result.output+ end++ # Records the detected embed on the collector and returns the+ # placeholder token. `source` is the verbatim matched slice; it is+ # stored on every row so the importer can restore the exact source text+ # when the embed can't be mapped, instead of rebuilding a canonical+ # form.+ def defer(node, source)+ case node+ when Markbridge::AST::Upload+ @embeds.upload(upload_id: node.sha1, original_markdown: source)+ when MarkdownScanner::UploadUrlReference+ ownership = upload_ownership(node)+ # A relative path outside the base prefix belongs to another+ # application on the same server. There is no host to allowlist, so+ # it is not recorded at all and the source text stays.+ return nil if ownership == :sibling_path++ @embeds.upload(+ upload_id: node.sha1,+ original_markdown: source,+ external_host: ownership,+ )+ when Markbridge::AST::Mention+ unless @mention_names.include?(Migrations::NameNormalizer.normalize(node.name))+ return nil+ end++ @embeds.mention(+ mention_type: @mention_classifier.call(node.name),+ name: node.name,+ original_markdown: source,+ )+ when MarkdownScanner::InternalLinkReference+ @embeds.link(+ url: node.url,+ text: node.text,+ target_type: LINK_TARGET_TYPES.fetch(node.target_type),+ target_id: node.target_id,+ target_name: node.target_name,+ target_topic_id: node.target_topic_id,+ target_post_number: node.target_post_number,+ target_tag_path: node.target_tag_path,+ target_suffix: node.target_suffix,+ original_markdown: source,+ url_offset: node.url_offset,+ label_url_offset: node.label_url_offset,+ )+ when MarkdownScanner::HashtagReference+ unless @hashtag_names.include?(Migrations::NameNormalizer.normalize(node.name))+ return nil+ end++ @embeds.hashtag(+ hashtag_type: FORCED_HASHTAG_TYPES[node.forced_type],+ name: node.name,+ original_markdown: source,+ )+ when MarkdownScanner::EmojiReference+ @embeds.emoji(name: node.name)+ when MarkdownScanner::QuoteReference+ defer_quote(node, source)+ else+ raise NotImplementedError, "no defer handler for #{node.class}"+ end+ end++ # Whether a full-URL upload is the source's own. Returns nil when it is;+ # the host string when it is not (the importer then maps its sha1 only+ # against an explicit allowlist); `:sibling_path` for a relative URL+ # outside the base prefix. The prefix matters on a subdirectory install:+ # `https://example.com/other/uploads/…` shares the forum's host but+ # belongs to a different application.+ def upload_ownership(node)+ if node.host+ return node.host unless @internal_link_hosts.key?(node.host)++ prefix = @internal_link_hosts[node.host]+ MarkdownScanner::UrlOrigin.path_within_prefix(node.rest, prefix).nil? ? node.host : nil+ elsif MarkdownScanner::UrlOrigin.path_within_prefix(+ node.rest,+ @internal_link_base_prefix,+ ).nil?+ :sibling_path+ end+ end++ # The converter never knows the quoted post's source `original_id`, so+ # it records the source coordinates instead. A quote with a `post:` but+ # no `topic:` points into its own topic; a `topic:` with no `post:`+ # drops both coordinates, because the importer can only resolve them as+ # a pair.+ def defer_quote(node, source)+ post_number = node.post_number+ topic_id = post_number ? (node.topic_id || @topic_id) : nil++ @embeds.quote(+ quoted_username: node.username,+ quoted_name: node.name,+ quoted_topic_id: topic_id,+ quoted_post_number: post_number,+ original_markdown: source,+ )+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rbindex 8ed78bf5b23..f36b7ed61d9 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb@@ -49,6 +49,7 @@ module Migrationspinned_globally: item[:pinned_globally],pinned_until: item[:pinned_until],slow_mode_seconds: item[:slow_mode_seconds],+ slug: item[:slug],subtype: item[:subtype],title: item[:title],user_id: item[:user_id],diff --git a/migrations/converters/lib/migrations/converters/embed_buffer.rb b/migrations/converters/lib/migrations/converters/embed_buffer.rbindex 20248af7ff1..cd9c0bb024e 100644--- a/migrations/converters/lib/migrations/converters/embed_buffer.rb+++ b/migrations/converters/lib/migrations/converters/embed_buffer.rb@@ -3,33 +3,28 @@module Migrationsmodule Converters# Collects the embeds found while an owner's body is converted to Markdown.- ## The owner is the record the markdown belongs to — a post today, a user bio- # or a group/category/badge description later. Its kind is fixed at- # construction (`owner_type`), so one buffer serves one kind of owner.- #- # For each embed it can't finish yet (a quote, link, mention, poll, event or- # upload), the Markdown converter calls the matching recorder here. We can't- # render the embed now because that needs the import maps, which only exist at- # import time. So the buffer mints a token, stores a descriptor that carries it,- # and returns the token to put into the raw in place of the embed.+ # or a group/category/badge description later — and its kind is fixed at+ # construction, so one buffer serves one kind of owner.#- # After the body is converted, the owning step writes the owner and then calls- # `write_for(owner_id)`, which inserts every recorded embed into its linkage- # table. A descriptor's keys match the table columns (minus `owner_type`/`owner_id`).+ # An embed cannot be rendered during conversion, because that needs the+ # import maps, which only exist at import time. So the converter calls the+ # matching recorder here, which mints a token, stores a descriptor carrying+ # it, and returns the token to put into the raw in place of the embed.+ # Afterwards the owning step writes the owner and calls+ # `write_for(owner_id)`. A descriptor's keys match the linkage table's+ # columns (minus `owner_type`/`owner_id`).#- # Recording embeds is pure (no database, no maps), so building a buffer is safe- # on the converter's worker threads. `write_for` is the only part that writes.+ # Recording is pure (no database, no maps), so building a buffer is safe on+ # the converter's worker threads. `write_for` is the only part that writes.class EmbedBufferIntermediateDB = Migrations::Database::IntermediateDBprivate_constant :IntermediateDBattr_reader :quotes, :links, :mentions, :hashtags, :emojis, :polls, :events, :uploads- # @param owner_type [Integer] the owning record's kind, an- # `IntermediateDB::Enums::EmbedOwner` value (e.g. `EmbedOwner::POST`).- # @param placeholder [Migrations::Placeholder] the token source; by default a- # fresh one (with its own nonce) per buffer.+ # @param placeholder [Migrations::Placeholder] the token source; by+ # default a fresh one, with its own nonce, per buffer.def initialize(owner_type:, placeholder: Migrations::Placeholder.new)@owner_type = owner_type@placeholder = placeholder@@ -43,28 +38,18 @@ module Migrations@uploads = []end- # The token replaces the opening `[quote="..."]` tag only. The Markdown- # converter writes the quoted text, the closing `[/quote]`, and the blank lines- # around them into the raw as plain text; none of that goes into the row.+ # The token replaces the opening `[quote="..."]` tag only; the quoted text+ # and the closing `[/quote]` stay in the raw as plain text. The quoted+ # post is identified by id or by coordinates, whichever the source gives —+ # post numbers are recomputed at import, so the importer resolves a+ # coordinate pair to a post before rendering.#- # The quoted post is identified either by id or by coordinates, whichever the- # source gives — the importer resolves the coordinates to a post before- # rendering (post numbers are recomputed at import).- #- # @param quoted_post_id [Integer, String, nil] the quoted post's source- # `original_id`, when the converter knows it.- # @param quoted_topic_id [Integer, String, nil] the quoted post's source- # topic id. Together with `quoted_post_number` it forms the source- # coordinates — the way the attribution identifies the post when- # `quoted_post_id` is unknown.- # @param quoted_post_number [Integer, nil] see `quoted_topic_id`.- # @param quoted_user_id [Integer, String, nil] the quoted user's source- # `original_id`.- # @param quoted_username [String, nil] the attribution's display fallback, used- # when the quoted user can't be mapped to a Discourse user; when- # `quoted_user_id` is nil, the importer also resolves it to the user.- # @param quoted_name [String, nil] like `quoted_username`, for sources that- # attribute quotes by full name.+ # @param quoted_username [String, nil] the header's display fallback; with+ # no `quoted_user_id`, the importer also resolves it to the user.+ # @param quoted_name [String, nil] for sources that name the quoted user+ # by full name.+ # @param original_markdown [String, nil] the verbatim source snippet,+ # restored unchanged when the importer cannot map the embed.# @return [String] the token for the opening tag.def quote(quoted_post_id: nil,@@ -72,7 +57,8 @@ module Migrationsquoted_post_number: nil,quoted_user_id: nil,quoted_username: nil,- quoted_name: nil+ quoted_name: nil,+ original_markdown: nil)record(@quotes,@@ -83,34 +69,29 @@ module Migrationsquoted_user_id:,quoted_username:,quoted_name:,+ original_markdown:,)end- # The target is identified in one of three forms, and only one is set per- # call: by id (`target_id`), by name (`target_name`), or by coordinates- # (`target_topic_id` + `target_post_number`). All stay nil for an external+ # Exactly one addressing form is set per call: by id, by name+ # (`target_name`: a username, group name, tag name or a `parent:child`+ # category slug path), or by coordinates. All stay nil for an external# link, which is just text carried through.#- # @param url [String, nil] the full source URL; the fallback whenever the- # target can't be resolved.- # @param text [String, nil] a markdown link's link text; nil for a bare URL,- # which is re-emitted bare to keep oneboxes working.- # @param target_type [Integer, nil] the kind of Discourse entity the link- # points at, an `IntermediateDB::Enums::LinkTarget` value (e.g.- # `LinkTarget::TOPIC`); nil for an external link.- # @param target_id [Integer, String, nil] the target's source `original_id` —- # for a topic, post, category or badge addressed by id in the URL.- # @param target_name [String, nil] the target's name — a username, group name,- # tag name, or a category slug path written `parent:child` — when the URL- # carries a name but no id.- # @param target_topic_id [Integer, String, nil] the source topic id of a post- # addressed as `/t/slug/<topic_id>/<post_number>`. Together with- # `target_post_number` it forms the source coordinates; post numbers are- # recomputed at import, so the importer resolves the pair.- # @param target_post_number [Integer, nil] see `target_topic_id`.- # @param target_suffix [String, nil] everything after the matched route- # (further path, query string, fragment), reattached verbatim when the URL- # is rebuilt.+ # @param url [String, nil] the full source URL; the fallback on a miss.+ # @param text [String, nil] a markdown link's link text; nil for a bare+ # URL, which is re-emitted bare to keep oneboxes working.+ # @param target_tag_path [String, nil] `[none/|all/]<tag>` for a+ # `CATEGORY_TAG` link (whose category rides in+ # `target_id`/`target_name`), `<t1>/<t2>[/…]` for a `TAG_INTERSECTION`+ # link.+ # @param target_suffix [String, nil] everything after the matched route,+ # reattached verbatim when the URL is rebuilt.+ # @param url_offset [Integer, nil] the destination's byte offset within+ # `original_markdown`, its span `url.bytesize` long — the importer+ # rewrites exactly that span rather than searching for the value.+ # @param label_url_offset [Integer, nil] the offset of a self-link label+ # spelling the destination too; nil otherwise.def link(url: nil,text: nil,@@ -119,7 +100,11 @@ module Migrationstarget_name: nil,target_topic_id: nil,target_post_number: nil,- target_suffix: nil+ target_tag_path: nil,+ target_suffix: nil,+ original_markdown: nil,+ url_offset: nil,+ label_url_offset: nil)record(@links,@@ -131,79 +116,69 @@ module Migrationstarget_name:,target_topic_id:,target_post_number:,+ target_tag_path:,target_suffix:,+ original_markdown:,+ url_offset:,+ label_url_offset:,)end- # @param mention_type [Integer, nil] an `IntermediateDB::Enums::MentionType`- # value (user, group, here or all); nil for a mention the converter- # couldn't classify, which the importer treats as a user mention.- # @param target_id [Integer, String, nil] the mentioned user's or group's- # source `original_id`; nil for `here`/`all`, or when only the name is- # known — the importer then resolves `name` to it.- # @param name [String, nil] the mention as written, without the leading `@`;- # the lookup key and the fallback text when the target can't be mapped.- # @raise [ArgumentError] if `mention_type` is neither nil nor a known type.- def mention(mention_type: nil, target_id: nil, name: nil)+ # @param mention_type [Integer, nil] an `Enums::MentionType` value; nil+ # for an unclassified mention, which the importer treats as a user+ # mention.+ # @param name [String, nil] the mention without the leading `@`; the+ # lookup key and the fallback text on a miss.+ # @raise [ArgumentError] on an unknown `mention_type`.+ def mention(mention_type: nil, target_id: nil, name: nil, original_markdown: nil)validate_mention_type!(mention_type)- record(@mentions, :mention, mention_type:, target_id:, name:)- end-- # @param hashtag_type [Integer, nil] an `IntermediateDB::Enums::HashtagType`- # value (category or tag). Set it when the source forced the type with a- # `::tag`/`::category` suffix or when `target_id` is given (an id renders- # only through its type); otherwise nil, and the importer classifies the- # name (categories first, then tags).- # @param target_id [Integer, String, nil] the source `original_id` of the- # category or tag, for a converter that identifies the target instead of- # just naming it; the importer then skips name resolution. Pin- # `hashtag_type` along with it.- # @param name [String, nil] the hashtag as written, without the leading `#`- # and any `::tag`/`::category` suffix; may hold one `:` as the- # `parent:child` category separator. Required even when `target_id` is- # set — it's the fallback text when the target can't be mapped at import.- # @raise [ArgumentError] if `hashtag_type` is neither nil nor a known type.- def hashtag(hashtag_type: nil, target_id: nil, name: nil)+ record(@mentions, :mention, mention_type:, target_id:, name:, original_markdown:)+ end++ # @param hashtag_type [Integer, nil] an `Enums::HashtagType` value. Set it+ # when the source forced the type with a `::tag`/`::category` suffix or+ # when `target_id` is given (an id renders only through its type);+ # otherwise nil, and the importer classifies the name.+ # @param name [String, nil] the hashtag without the leading `#` and any+ # `::tag`/`::category` suffix; may hold one `:` as the `parent:child`+ # separator. Required even with `target_id` — the fallback text on a+ # miss.+ # @raise [ArgumentError] on an unknown `hashtag_type`.+ def hashtag(hashtag_type: nil, target_id: nil, name: nil, original_markdown: nil)validate_hashtag_type!(hashtag_type)- record(@hashtags, :hashtag, hashtag_type:, target_id:, name:)+ record(@hashtags, :hashtag, hashtag_type:, target_id:, name:, original_markdown:)end- # Only the source's own custom emoji reach here; a standard shortcode stays- # plain text.- #- # @param name [String, nil] the shortcode without the surrounding colons.+ # No `original_markdown` snippet: an emoji embed only exists when its+ # `:name:` bytes occur in the source, so the importer rebuilds the exact+ # source spelling from the name alone.def emoji(name: nil)record(@emojis, :emoji, name:)end- # @param poll_id [Integer, String, nil] the poll's source `original_id` (a- # `polls` row converted by its own step); the importer renders that poll- # into the raw.def poll(poll_id: nil)record(@polls, :poll, poll_id:)end- # @param event_id [Integer, String, nil] the event's source `original_id` (an- # `events` row converted by its own step); the importer renders that event- # into the raw.def event(event_id: nil)record(@events, :event, event_id:)end- # @param upload_id [String, nil] the referenced `uploads` row's id — a content- # hash, so text rather than numeric.- # @param original_markdown [String, nil] the verbatim source snippet for an- # upload referenced by a full URL, so the importer can put it back when the- # sha1 maps to no Discourse upload. Stays nil for a short `upload://`- # reference, which has no meaningful fallback.- def upload(upload_id: nil, original_markdown: nil)- record(@uploads, :upload, upload_id:, original_markdown:)+ # @param upload_id [String, nil] the referenced `uploads` row's id — a+ # content hash, so text rather than numeric. The Discourse extractor+ # records the 40-hex sha1 for full-URL forms and the base62 short id for+ # `upload://` references, so the importer's map must answer both.+ # @param external_host [String, nil] for a full-URL upload on a host that+ # is not the source's own: the importer maps its id only when the+ # conversion allowlists that host, so a foreign URL whose basename+ # collides with a source upload sha1 is restored verbatim.+ def upload(upload_id: nil, original_markdown: nil, external_host: nil)+ record(@uploads, :upload, upload_id:, original_markdown:, external_host:)end- # Empties the recorded embeds (in place, keeping the collections) so one buffer- # can be reused for the next owner instead of allocating a fresh one — and its- # placeholder along with it — per owner. The placeholder is kept: its running- # sequence is what keeps tokens unique across the owners that share the buffer.+ # Empties the recorded embeds in place, so one buffer can serve the next+ # owner. The placeholder is kept: its running sequence is what keeps+ # tokens unique across the owners that share a buffer.def clear@quotes.clear@links.clear@@ -216,11 +191,8 @@ module Migrationsselfend- # Inserts each recorded embed into its linkage table. Call once per owner, after- # the owner row is written.- #- # @param owner_id [Integer, String] the owner's source `original_id`, matching- # the id the owner row was written with.+ # Call once per owner, after the owner row is written. `owner_id` is the+ # owner's source `original_id`.def write_for(owner_id)owner_type = @owner_type@quotes.each { |row| IntermediateDB::EmbedQuote.create(owner_type:, owner_id:, **row) }@@ -233,8 +205,8 @@ module Migrations@uploads.each { |row| IntermediateDB::EmbedUpload.create(owner_type:, owner_id:, **row) }end- # @return [Array<String>] every token created, in order (used to assert they- # all reached the raw).+ # @return [Array<String>] every token created, in order (used to assert+ # they all reached the raw).def placeholdersdescriptors.map { |descriptor| descriptor[:placeholder] }enddiff --git a/migrations/converters/lib/migrations/converters/markdown_engine.rb b/migrations/converters/lib/migrations/converters/markdown_engine.rbnew file mode 100644index 00000000000..2872479c36d--- /dev/null+++ b/migrations/converters/lib/migrations/converters/markdown_engine.rb@@ -0,0 +1,20 @@+# frozen_string_literal: true++module Migrations+ module Converters+ # A self-contained discourse-markdown-it engine for scanning post markdown+ # during a conversion: the same parser, features, and options the target+ # site will use, but without a booted Rails application or a local site+ # database. Parse-only — it never renders, sanitizes, or resolves+ # render-time data.+ #+ # {MarkdownEngine::Bundle} prepares all checkout-derived JavaScript once, in+ # the parent process; {MarkdownEngine::Context} is the per-worker V8 wrapper+ # built from a bundle and a {MarkdownEngine::Config} of source-site inputs.+ module MarkdownEngine+ def self.discourse_root+ Migrations.host_app_root+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/markdown_engine/bundle.rb b/migrations/converters/lib/migrations/converters/markdown_engine/bundle.rbnew file mode 100644index 00000000000..dbf7facaa44--- /dev/null+++ b/migrations/converters/lib/migrations/converters/markdown_engine/bundle.rb@@ -0,0 +1,283 @@+# frozen_string_literal: true++require "digest"+require "fileutils"+require "json"+require "open3"++module Migrations+ module Converters+ module MarkdownEngine+ # The ordered JavaScript a `Context` evaluates: the host application's+ # precompiled pretty-text bundle (built out of process by+ # `frontend/pretty-text-processor`), the supported plugins' vendored+ # libraries and markdown features (the latter transpiled through the+ # host's `AssetProcessor`), and the generated emoji replacement table.+ #+ # Cached on disk keyed by a digest of every input. On a cache miss the+ # build runs in a SUBPROCESS that writes the cache and exits, because+ # transpiling the plugin features boots `AssetProcessor`'s own V8 and V8+ # initialized multithreaded is not fork-safe — the converter parent forks+ # workers, so it must never hold V8 state, nor the Rails/Discourse+ # stand-ins the host build code needs ({HostShims}). A warm cache needs+ # neither node nor pnpm.+ class Bundle+ class BuildError < StandardError+ end++ # Bump when the entry list or generation logic changes in a way the+ # input files cannot express.+ VERSION = 2++ CACHE_DIR = "tmp/migrations"++ # What the bundled plugins register as `:vendored_pretty_text` /+ # `:vendored_core_pretty_text` assets; plugin feature modules reference+ # these as plain globals.+ PLUGIN_VENDOR_FILES = %w[+ frontend/discourse/node_modules/moment/moment.js+ frontend/discourse/node_modules/moment-timezone/builds/moment-timezone-with-data.js+ plugins/footnote/assets/vendor/javascripts/markdown-it-footnote.js+ ].freeze++ # Every plugin bundled with the host application whose markdown features+ # change tokenization. Their constructs must tokenize the way the+ # destination will see them, or text inside e.g. a `[poll]` or `$$…$$`+ # would be scanned as ordinary prose and a mention in there would be+ # replaced. The set is fixed, not configuration: `Config::SETTING_KEYS`+ # covers exactly these plugins' parse-relevant settings, so a plugin+ # outside the list would run its markdown rules with absent settings. A+ # drift spec compares this list against the plugins that actually ship+ # markdown files. discourse-ai is the one deliberate exclusion: its+ # markdown modules only allowlist HTML attributes and register no rules.+ CORE_MARKDOWN_PLUGINS = %w[+ chat+ checklist+ discourse-details+ discourse-events+ discourse-graphviz+ discourse-local-dates+ discourse-math+ discourse-policy+ footnote+ poll+ spoiler-alert+ ].freeze++ # Digesting reads the host constants and file globs only, so no V8 boots+ # here and the host classes stay Rails-free.+ def self.load_or_build(cache_dir: nil)+ root = MarkdownEngine.discourse_root+ cache_dir ||= File.join(root, CACHE_DIR)+ # rubocop:disable Discourse/NoChdir+ Dir.chdir(root) do+ require_host_build_classes(root)++ digest = input_digest(root)+ cache_file = File.join(cache_dir, "markdown-engine-bundle-#{digest}.json")+ entries = read_cache(cache_file)+ return new(entries) if entries++ FileUtils.mkdir_p(cache_dir)+ File.open(File.join(cache_dir, "bundle.lock"), File::CREAT | File::RDWR) do |lock|+ lock.flock(File::LOCK_EX)+ # Another process may have built while this one waited.+ entries = read_cache(cache_file)+ unless entries+ build_in_subprocess(root, cache_file)+ entries = read_cache(cache_file)+ if entries.nil?+ raise BuildError, "bundle build subprocess produced no readable cache"+ end+ end+ cleanup_stale_caches(cache_dir, cache_file)+ end+ new(entries)+ end+ # rubocop:enable Discourse/NoChdir+ end++ # Run this only in the separate process that `load_or_build` spawns, for+ # the reason in the class comment. Writes via temp file and atomic+ # rename, so a crashed build cannot leave a truncated cache another run+ # trusts.+ def self.build_and_write(root, cache_file)+ # discourse-emojis decides whether to load its railtie by checking for+ # `Rails`, so it must load before the stand-in exists.+ require "discourse_emojis"+ require "mini_racer"+ HostShims.install!(root)+ # rubocop:disable Discourse/NoChdir+ Dir.chdir(root) do+ require_host_build_classes(root)++ entries = build_entries(root)+ FileUtils.mkdir_p(File.dirname(cache_file))+ temp_file = "#{cache_file}.#{Process.pid}.tmp"+ begin+ File.write(temp_file, JSON.generate({ "entries" => entries }))+ File.rename(temp_file, cache_file)+ ensure+ File.delete(temp_file) if File.exist?(temp_file)+ end+ end+ # rubocop:enable Discourse/NoChdir+ end++ # A missing file, a truncated or corrupt one, and valid JSON with the+ # wrong shape are all cache misses; without the shape check,+ # `{"entries":"corrupt"}` would reach the context and raise there+ # instead of rebuilding. An empty list is a miss too: no build produces+ # zero entries.+ def self.read_cache(cache_file)+ entries = JSON.parse(File.read(cache_file))["entries"]+ return nil unless entries.is_a?(Array)+ return nil if entries.empty?+ return nil unless entries.all? { |entry| entry_pair?(entry) }++ entries+ rescue Errno::ENOENT, JSON::ParserError+ nil+ end++ def self.entry_pair?(entry)+ entry.is_a?(Array) && entry.size == 2 && entry.all?(String)+ end++ def self.build_in_subprocess(root, cache_file)+ program =+ "require \"migrations-converters\"; " \+ "Migrations::Converters::MarkdownEngine::Bundle.build_and_write(ARGV[0], ARGV[1])"+ _stdout, stderr, status =+ Open3.capture3(RbConfig.ruby, "-e", program, root, cache_file, chdir: root)+ return if status.success?++ raise BuildError, "bundle build subprocess failed: #{stderr.strip}"+ end++ # @return [Array<Array(String, String)>] `[module_name, source]` pairs+ attr_reader :entries++ def initialize(entries)+ @entries = entries+ end++ # `AssetProcessor` references `PrecompiledBundle` in its class body, so+ # the load order matters. A booted application has both already.+ def self.require_host_build_classes(root)+ return if Object.const_defined?(:AssetProcessor)++ require File.join(root, "lib", "precompiled_bundle")+ require File.join(root, "lib", "asset_processor")+ require File.join(root, "lib", "pretty_text", "core_bundle")+ end++ def self.input_digest(root)+ digest = Digest::MD5.new+ digest.update("v#{VERSION}")+ digest.update("compiler-v#{::AssetProcessor::BASE_COMPILER_VERSION}")+ digest_globs(digest, root, ::AssetProcessor::BUNDLE.dependency_globs)+ digest_globs(digest, root, core_bundle_globs)++ (input_files(root) + EmojiData.data_files).each do |path|+ digest.update(path)+ digest.update(File.read(path))+ end+ digest.hexdigest+ end++ def self.digest_globs(digest, root, globs)+ globs.each do |pattern|+ Dir+ .glob(pattern, base: root)+ .sort+ .each do |file|+ digest.update(file)+ digest.update(File.read(File.join(root, file)))+ end+ end+ end++ def self.input_files(root)+ files = PLUGIN_VENDOR_FILES.map { |path| File.join(root, path) }+ CORE_MARKDOWN_PLUGINS.each { |plugin| files.concat(plugin_files(root, plugin)) }+ files+ end++ def self.plugin_files(root, plugin)+ pattern =+ File.join(+ root,+ "plugins",+ plugin,+ "assets/javascripts/**/discourse-markdown/**/*.{js,js.es6}",+ )+ Dir[pattern].sort.filter { |path| File.file?(path) }+ end++ # The host's own definition, so there is one bundle: a booted+ # application shares this object with `PrettyText` and the same cache+ # file; standalone, the host stand-ins carry its build block.+ def self.host_core_bundle+ require_host_build_classes(MarkdownEngine.discourse_root)+ ::PrettyText::CoreBundle::BUNDLE+ end++ # @return [Array<String>] the dependency globs of the host core bundle,+ # the JavaScript sources the produced context is built from+ def self.core_bundle_globs+ host_core_bundle.dependency_globs+ end++ def self.build_entries(root)+ entries = [["pretty-text.js", host_core_bundle.load_or_build]]++ PLUGIN_VENDOR_FILES.each { |path| entries << [path, File.read(File.join(root, path))] }++ CORE_MARKDOWN_PLUGINS.each do |plugin|+ plugin_files(root, plugin).each do |path|+ # Feature discovery only pattern-matches the `discourse/plugins/`+ # prefix and the file name, so the directory name can stand in for+ # the plugin's registered name.+ module_name =+ path.sub(%r{\A.+assets/javascripts/}, "discourse/plugins/#{plugin}/").sub(+ /\.js(\.es6)?\z/,+ "",+ )+ entries << transpiled_entry(path, module_name)+ end+ end++ entries << ["migrations/emoji-data", EmojiData.set_unicode_source]+ entries+ end++ def self.transpiled_entry(path, module_name)+ source = File.read(path)+ [module_name, ::AssetProcessor.new.perform(source, nil, module_name)]+ end++ # Runs under the build lock, so no concurrent starter can be reading a+ # stale file as it disappears.+ def self.cleanup_stale_caches(cache_dir, current_file)+ Dir[File.join(cache_dir, "markdown-engine-bundle-*.json")].each do |path|+ File.delete(path) if path != current_file+ end+ end++ private_class_method :input_digest,+ :digest_globs,+ :input_files,+ :plugin_files,+ :host_core_bundle,+ :build_entries,+ :transpiled_entry,+ :entry_pair?,+ :cleanup_stale_caches,+ :read_cache,+ :build_in_subprocess,+ :require_host_build_classes+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/markdown_engine/config.rb b/migrations/converters/lib/migrations/converters/markdown_engine/config.rbnew file mode 100644index 00000000000..a98371e77a7--- /dev/null+++ b/migrations/converters/lib/migrations/converters/markdown_engine/config.rb@@ -0,0 +1,156 @@+# frozen_string_literal: true++require "yaml"++module Migrations+ module Converters+ module MarkdownEngine+ # The source-site inputs a `Context` is built from. Site settings start+ # from the checkout's own YAML defaults and are overridden by whatever the+ # source database provides for the keys the markdown pipeline reads; a+ # spec keeps `SETTING_KEYS` in sync with the JavaScript by grepping the+ # same files the `Bundle` loads. The keys cover core and exactly the+ # plugins in `Bundle::CORE_MARKDOWN_PLUGINS`.+ class Config+ # Every `siteSettings.*` key the loaded markdown JavaScript reads.+ SETTING_KEYS = %w[+ allowed_href_schemes+ allowed_iframes+ chat_enabled+ checklist_enabled+ default_code_lang+ discourse_events_enabled+ discourse_graphviz_enabled+ discourse_local_dates_email_format+ discourse_local_dates_email_timezone+ discourse_local_dates_enabled+ discourse_math_enable_asciimath+ discourse_math_enable_latex_delimiters+ discourse_math_enabled+ discourse_math_provider+ discourse_post_event_enabled+ emoji_set+ enable_emoji+ enable_emoji_shortcuts+ enable_inline_emoji_translation+ enable_markdown_footnotes+ enable_markdown_linkify+ enable_markdown_typographer+ enable_mentions+ external_emoji_url+ markdown_linkify_tlds+ markdown_typographer_quotation_marks+ policy_enabled+ poll_enabled+ poll_maximum_options+ secure_uploads+ spoiler_enabled+ traditional_markdown_linebreaks+ unicode_usernames+ ].freeze++ # Read from the options directly, not through `siteSettings`.+ AVATAR_SIZES_KEY = "avatar_sizes"++ attr_reader :settings, :category_slugs, :tag_names, :additional_options++ # @param source_settings [Hash] the source site's settings; only+ # `SETTING_KEYS` are used. String values for boolean/integer settings+ # are coerced against the default's type.+ # @param category_slugs [Array<String>] the source's category slug+ # paths, a nested category as `parent:child`+ def initialize(+ source_settings: {},+ category_slugs: [],+ tag_names: [],+ custom_emoji_names: [],+ additional_options: {}+ )+ defaults = self.class.default_settings+ overrides =+ source_settings+ .transform_keys(&:to_s)+ .slice(*SETTING_KEYS)+ .to_h { |key, value| [key, self.class.coerce(value, defaults[key])] }+ @settings = defaults.merge(overrides)+ # Injected already {NameNormalizer}-folded; runtime.js folds the+ # sought slug the same way.+ @category_slugs = category_slugs.map { |slug| NameNormalizer.normalize(slug) }+ @tag_names = tag_names.map { |name| NameNormalizer.normalize(name) }+ @custom_emoji_names = custom_emoji_names.map(&:to_s)+ @additional_options = additional_options+ end++ # The lookup set runtime.js answers `#slug` against: a category is+ # addressed by its own slug, so a nested category enters as its leaf.+ # The extractor still validates the parent path against+ # {#hashtag_names}.+ def category_lookup_slugs+ @category_lookup_slugs ||=+ @category_slugs.map { |path| path.split(":").last }.compact.uniq+ end++ # Every hashtag name the source has, in the form a post spells it (a+ # nested category as the full path) — what a `RawExtractor` resolves+ # against.+ def hashtag_names+ @hashtag_names ||= CompactStringSet.new(@category_slugs + @tag_names)+ end++ def avatar_sizes+ self.class.yaml_defaults[AVATAR_SIZES_KEY].to_s.split("|").map(&:to_i)+ end++ def custom_emoji+ # Only the names matter for scanning; the URL just has to be a+ # plausible path for token construction.+ @custom_emoji_names.to_h { |name| [name, "/images/emoji/custom/#{name}.png"] }+ end++ def self.default_settings+ yaml_defaults.slice(*SETTING_KEYS)+ end++ def self.yaml_defaults+ @yaml_defaults ||=+ begin+ root = MarkdownEngine.discourse_root+ files =+ [File.join(root, "config", "site_settings.yml")] ++ Bundle::CORE_MARKDOWN_PLUGINS.filter_map do |plugin|+ path = File.join(root, "plugins", plugin, "config", "settings.yml")+ path if File.exist?(path)+ end++ defaults = {}+ files.each do |file|+ YAML+ .safe_load(File.read(file), aliases: true)+ .each_value do |group|+ next unless group.is_a?(Hash)+ group.each do |name, value|+ next unless SETTING_KEYS.include?(name) || name == AVATAR_SIZES_KEY+ defaults[name] = value.is_a?(Hash) ? value["default"] : value+ end+ end+ end+ defaults+ end+ end++ def self.coerce(value, default)+ return value unless value.is_a?(String)++ case default+ when true, false+ %w[t true].include?(value.downcase)+ when Integer+ value.to_i+ else+ value+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/markdown_engine/context.rb b/migrations/converters/lib/migrations/converters/markdown_engine/context.rbnew file mode 100644index 00000000000..56246ca233d--- /dev/null+++ b/migrations/converters/lib/migrations/converters/markdown_engine/context.rb@@ -0,0 +1,259 @@+# frozen_string_literal: true++require "json"+require "mini_racer"++module Migrations+ module Converters+ module MarkdownEngine+ # One V8 engine per worker process. V8 contexts do not survive forking, so+ # a context is created after fork (a worker step's `setup`) and a fork+ # hook discards any instance a child accidentally inherits.+ class Context+ class DiscardedError < StandardError+ end++ # PrettyText allows 25s because it renders; a scan only parses. The+ # slowest legitimate parse we measured on a large corpus was 435 ms, so+ # 3 seconds has plenty of headroom while it keeps one runaway body from+ # dominating V8 time. A body the ceiling cuts off gets one retry with a+ # larger ceiling (`EngineScanner::SLOW_TIMEOUT_MS`), not a refusal.+ EVAL_TIMEOUT_MS = 3_000++ # @return [Config] the source-site inputs this engine was built from,+ # for a caller that needs the name sets beside the engine+ attr_reader :config++ def initialize(bundle:, config:)+ @bundle = bundle+ @config = config+ @timeout_ms = EVAL_TIMEOUT_MS+ @needs_rebuild = false+ @monitor = Mutex.new+ @watchdog_signal = ConditionVariable.new+ @fork_hook = ForkManager.after_fork_child { discard! }+ build_context(@timeout_ms)+ end++ # @param posts [Array<Hash>] `{ id:, raw: }` per post+ # @param timeout_ms [Integer, nil] a one-off ceiling for this scan.+ # MiniRacer fixes an isolate's timeout at construction, so the isolate+ # keeps the widest ceiling it was asked for (a rebuild costs about+ # 0.15s) and anything below that is enforced by {#with_ceiling} — a+ # caller counting a per-body deadline down must not pay for a rebuild+ # per call.+ # @return [Array<Hash>] per-post block/construct data from scan.js+ def scan(posts, timeout_ms: nil)+ wanted = timeout_ms || @timeout_ms+ if @context.nil?+ raise DiscardedError unless @needs_rebuild+ build_context(wanted)+ elsif wanted > @ceiling_ms+ dispose_context+ build_context(wanted)+ end++ payload =+ posts.map do |post|+ raw = post[:raw].to_s+ raw = raw.scrub unless raw.valid_encoding?+ { "id" => post[:id], "raw" => raw }+ end+ with_ceiling(wanted) { @context.call("__scanPosts", payload) }+ end++ # A timeout can leave arbitrary JS state behind, so the context goes and+ # is rebuilt lazily on the next scan.+ def reset!+ dispose_context+ @needs_rebuild = true+ end++ def close+ dispose_context+ @needs_rebuild = false+ stop_watchdog++ if @fork_hook+ ForkManager.remove_after_fork_child(@fork_hook)+ @fork_hook = nil+ end+ end++ # Runs in a freshly forked child: the inherited isolate belongs to the+ # parent and must not be used or disposed from here, so only the+ # reference is dropped — and no lazy rebuild either, a worker builds its+ # own context deliberately.+ def discard!+ @context = nil+ @needs_rebuild = false+ # Threads do not survive a fork, and a lock the watchdog held while+ # the parent forked would still be held here.+ @monitor = Mutex.new+ @watchdog_signal = ConditionVariable.new+ @watchdog = nil+ @deadline = nil+ end++ private++ def build_context(timeout_ms)+ @context = MiniRacer::Context.new(timeout: timeout_ms, ensure_gc_after_idle: 2000)+ @ceiling_ms = timeout_ms+ @needs_rebuild = false+ evaluate_all(@bundle, @config)+ end++ # Disposal races the watchdog thread, which must not reach for a context+ # that is going away.+ def dispose_context+ @monitor.synchronize do+ @context&.dispose+ @context = nil+ end+ end++ # A ceiling below the isolate's own is enforced from a helper thread:+ # `MiniRacer::Context#stop` terminates the running script the way V8's+ # own watchdog does, so the caller sees the same `ScriptTerminatedError`+ # either way.+ def with_ceiling(timeout_ms)+ return yield if timeout_ms >= @ceiling_ms++ arm_watchdog(timeout_ms)+ terminated = false+ begin+ yield+ rescue MiniRacer::ScriptTerminatedError+ terminated = true+ raise+ ensure+ # The watchdog can fire between the script finishing and the disarm,+ # and V8 keeps an undelivered termination request for its next entry+ # — so the isolate goes rather than a later scan being cut short by+ # it.+ reset! if disarm_watchdog && !terminated+ end+ end++ def arm_watchdog(timeout_ms)+ @monitor.synchronize do+ @deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout_ms / 1000.0+ @fired = false+ @watchdog ||= Thread.new { watch }+ @watchdog_signal.signal+ end+ end++ # @return [Boolean] whether the watchdog terminated the script+ def disarm_watchdog+ @monitor.synchronize do+ @deadline = nil+ @fired+ end+ end++ def stop_watchdog+ watchdog = nil+ @monitor.synchronize do+ @closing = true+ watchdog = @watchdog+ @watchdog = nil+ @watchdog_signal.signal+ end+ watchdog&.join+ end++ # Sleeps until the armed deadline, an earlier disarm, or `close`. The+ # monitor is held only between waits, never while a script runs.+ def watch+ @monitor.synchronize do+ until @closing+ if @deadline.nil?+ @watchdog_signal.wait(@monitor)+ next+ end++ remaining = @deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)+ if remaining > 0+ @watchdog_signal.wait(@monitor, remaining)+ else+ @deadline = nil+ @fired = true+ begin+ @context&.stop+ rescue MiniRacer::ContextDisposedError+ # Raced a disposal that beat this thread to the monitor.+ nil+ end+ end+ end+ end+ end++ def evaluate_all(bundle, config)+ # The environment the loaded modules expect.+ @context.eval(<<~JS, filename: "migrations/prelude.js")+ window = globalThis;+ window.devicePixelRatio = 2;+ console = { log() {}, warn() {}, error() {} };+ __PRETTY_TEXT = true;+ JS++ # The pretty-text bundle captures `globalThis.__Ruby` while it+ # evaluates, so the scan-mode stubs must exist before the entries.+ @context.eval(+ File.read(File.join(__dir__, "runtime.js")),+ filename: "migrations/runtime.js",+ )+ bundle.entries.each { |name, source| @context.eval(source, filename: name) }++ @context.eval(<<~JS, filename: "migrations/scan-config.js")+ __scanConfig = {+ categorySlugs: #{config.category_lookup_slugs.to_h { |slug| [slug, true] }.to_json},+ tagNames: #{config.tag_names.to_h { |name| [name, true] }.to_json}+ };+ JS++ @context.eval(options_source(config), filename: "migrations/options.js")+ @context.eval(File.read(File.join(__dir__, "scan.js")), filename: "migrations/scan.js")+ @context.low_memory_notification+ end++ # Mirrors the option construction in `PrettyText.markdown`, with+ # scan-mode values: no watched words (target-site config, unknown at+ # conversion time), no censoring, empty paths. `__PrettyText.cook`+ # installs its callback surface and the unicode replacer into the+ # options it is given, so priming it with an empty string reuses that+ # assembly.+ def options_source(config)+ <<~JS+ __optInput = {+ siteSettings: #{config.settings.to_json},+ allowedIframes: #{config.settings["allowed_iframes"].to_s.split("|").to_json},+ paths: { baseUri: "" },+ customEmoji: #{config.custom_emoji.to_json},+ customEmojiTranslation: {},+ emojiDenyList: [],+ censoredRegexp: [],+ watchedWordsReplace: null,+ watchedWordsLink: null,+ additionalOptions: #{config.additional_options.to_json},+ avatar_sizes: #{config.avatar_sizes.to_json},+ hashtagTypesInPriorityOrder: ["category", "tag"],+ hashtagIcons: { category: "folder", tag: "tag" }+ };+ __PrettyText.cook("", __optInput);+ __pt = require("discourse-markdown-it").default+ .withCustomFeatures(require("discourse/static/markdown-it/features").default())+ .withOptions(__optInput);+ // A reference definition's destination is a construct occurrence the+ // scan has to count, and this rule drops the only token that says+ // where that occurrence is.+ __pt.options.engine.core.ruler.disable("strip_references");+ JS+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/markdown_engine/emoji_data.rb b/migrations/converters/lib/migrations/converters/markdown_engine/emoji_data.rbnew file mode 100644index 00000000000..3c846084fb2--- /dev/null+++ b/migrations/converters/lib/migrations/converters/markdown_engine/emoji_data.rb@@ -0,0 +1,85 @@+# frozen_string_literal: true++require "discourse_emojis"+require "json"++module Migrations+ module Converters+ module MarkdownEngine+ # The unicode-emoji-to-name replacement table `__PrettyText.setUnicode`+ # expects. Mirrors `Emoji.unicode_replacements` in the host application,+ # which cannot be required here because the `Emoji` model drags in site+ # caches and settings; a parity spec asserts the outputs stay identical.+ module EmojiData+ FITZPATRICK_SCALE = %w[1f3fb 1f3fc 1f3fd 1f3fe 1f3ff].freeze++ # Kept as text symbols rather than emoji images by the host application.+ SYMBOL_NAMES = %w[registered copyright trade_mark left_right_arrow].freeze++ EXTRA_REPLACEMENTS = {+ "\u{2639}" => "frowning",+ "\u{263B}" => "slight_smile",+ "\u{2661}" => "heart",+ "\u{2665}" => "heart",+ }.freeze++ def self.unicode_replacements+ @unicode_replacements ||=+ begin+ replacements = {}+ tonable = tonable_emojis+ scales = FITZPATRICK_SCALE.map { |scale| scale.to_i(16) }++ emojis_db.each do |emoji|+ name = emoji["name"]+ next if SYMBOL_NAMES.include?(name)++ code = replacement_code(emoji["code"])+ next unless code++ replacements[code] = name+ if tonable.include?(name)+ scales.each_with_index do |scale, index|+ codepoints = code.codepoints+ codepoints.delete_at(1) if codepoints[1] == 0xfe0f+ toned_code = codepoints.insert(1, scale).pack("U*")+ replacements[toned_code] = "#{name}:t#{index + 2}"+ end+ end+ end++ replacements.merge(EXTRA_REPLACEMENTS)+ end+ end++ def self.set_unicode_source+ "__PrettyText.setUnicode(#{unicode_replacements.to_json});"+ end++ def self.data_files+ [DiscourseEmojis.paths[:emojis], DiscourseEmojis.paths[:tonable_emojis]]+ end++ def self.emojis_db+ parse_file(DiscourseEmojis.paths[:emojis])+ end+ private_class_method :emojis_db++ def self.tonable_emojis+ parse_file(DiscourseEmojis.paths[:tonable_emojis])+ end+ private_class_method :tonable_emojis++ def self.parse_file(path)+ File.open(path, "r:UTF-8") { |file| JSON.parse(file.read) }+ end+ private_class_method :parse_file++ def self.replacement_code(code)+ code.split("-").map!(&:hex).pack("U*")+ end+ private_class_method :replacement_code+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/markdown_engine/host_shims.rb b/migrations/converters/lib/migrations/converters/markdown_engine/host_shims.rbnew file mode 100644index 00000000000..33f0352344a--- /dev/null+++ b/migrations/converters/lib/migrations/converters/markdown_engine/host_shims.rb@@ -0,0 +1,72 @@+# frozen_string_literal: true++require "fileutils"+require "logger"+require "open3"+require "pathname"++module Migrations+ module Converters+ module MarkdownEngine+ # The host application's build classes (`PrecompiledBundle` for the+ # pretty-text bundle, `AssetProcessor` for plugin-feature transpiles)+ # reach for `Rails.root`/`Rails.logger`, `GlobalSetting`, and+ # `Discourse::Utils`. A converter's build subprocess has none of them, and+ # booting the application to get them is what this component avoids, so+ # minimal stand-ins go in instead. Inside a booted application every+ # constant is already defined and nothing here runs.+ module HostShims+ def self.install!(discourse_root)+ install_rails(discourse_root) unless defined?(::Rails)+ install_global_setting unless defined?(::GlobalSetting)+ install_discourse unless defined?(::Discourse)+ end++ def self.install_rails(discourse_root)+ root = Pathname.new(discourse_root)+ logger = Logger.new(IO::NULL)+ rails = Module.new+ rails.define_singleton_method(:root) { root }+ rails.define_singleton_method(:logger) { logger }+ Object.const_set(:Rails, rails)+ end+ private_class_method :install_rails++ def self.install_global_setting+ global_setting = Module.new+ global_setting.define_singleton_method(:mini_racer_single_threaded) { false }+ Object.const_set(:GlobalSetting, global_setting)+ end+ private_class_method :install_global_setting++ def self.install_discourse+ utils = Module.new+ utils.define_singleton_method(:execute_command) do |*command, chdir: nil|+ options = chdir ? { chdir: } : {}+ stdout, stderr, status = Open3.capture3(*command, **options)+ raise "Command failed: #{command.join(" ")}\n#{stderr}" unless status.success?+ stdout+ end+ utils.define_singleton_method(:atomic_write_file) do |destination, contents|+ existing =+ begin+ File.read(destination)+ rescue Errno::ENOENT+ nil+ end+ unless existing == contents+ FileUtils.mkdir_p(File.dirname(destination))+ temp_destination = "#{destination}.#{Process.pid}.tmp"+ File.write(temp_destination, contents)+ File.rename(temp_destination, destination)+ end+ end+ discourse = Module.new+ discourse.const_set(:Utils, utils)+ Object.const_set(:Discourse, discourse)+ end+ private_class_method :install_discourse+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/markdown_engine/runtime.js b/migrations/converters/lib/migrations/converters/markdown_engine/runtime.jsnew file mode 100644index 00000000000..1b65efd60f7--- /dev/null+++ b/migrations/converters/lib/migrations/converters/markdown_engine/runtime.js@@ -0,0 +1,82 @@+/* eslint-disable no-undef, no-unused-vars */+// Scan-mode replacements for the Ruby helper methods PrettyText attaches as+// `__Ruby.*`. The pretty-text bundle captures `globalThis.__Ruby` when it is+// evaluated, so this must run first. Render-oriented lookups are inert; the+// hashtag lookup — which decides whether a `#slug` produces a token at all —+// answers from source-site data injected as `__scanConfig`.++__Ruby = {+ // The key is deterministic and never reaches the token data.+ t(key) {+ return key;+ },+ avatar_template() {+ return "";+ },+ lookup_primary_user_group() {+ return null;+ },+ format_username(username) {+ return username;+ },+ // Leaving every short URL unresolved makes the upload protocol store the+ // original in data-orig-*, which is the value the scan extracts.+ lookup_upload_urls() {+ return {};+ },+ get_topic_info() {+ return null;+ },+ get_current_user() {+ return null;+ },+ hashtag_lookup(slug, cookingUserId, typesInPriorityOrder) {+ // Ruby lowercases Σ to σ regardless of position. Map the capital first+ // to avoid JavaScript's contextual mapping, preserving literal ς.+ const ref = slug+ .normalize("NFC")+ .replace(/\u03a3/g, "\u03c3")+ .toLowerCase();+ // `#slug::type` forces one type; `#parent:child` addresses a category by+ // its child slug; `ref` keeps the typed form, like the host lookup service.+ // Only slug, type and ref shape the tokens a scan reads.+ const [name, forcedType] = ref.split("::");+ const types = forcedType ? [forcedType] : typesInPriorityOrder;++ for (const type of types) {+ if (type === "category") {+ // Ruby's split(":") drops trailing empty segments, so core resolves a+ // dangling `#general:` to the `general` category; JS split keeps the+ // empty tail, hence the filter.+ const slugPart =+ name+ .split(":")+ .filter((part) => part !== "")+ .pop() || "";+ if (slugPart !== "" && __scanConfig.categorySlugs[slugPart]) {+ // `text` becomes the rendered label, which no scan reads.+ return {+ relative_url: "/c/" + slugPart,+ text: slugPart,+ type: "category",+ slug: slugPart,+ id: 0,+ ref,+ };+ }+ } else if (type === "tag") {+ if (__scanConfig.tagNames[name]) {+ return {+ relative_url: "/tag/" + name,+ text: name,+ type: "tag",+ slug: name,+ id: 0,+ ref,+ };+ }+ }+ }+ return null;+ },+};diff --git a/migrations/converters/lib/migrations/converters/markdown_engine/scan.js b/migrations/converters/lib/migrations/converters/markdown_engine/scan.jsnew file mode 100644index 00000000000..1e4c16e4dd2--- /dev/null+++ b/migrations/converters/lib/migrations/converters/markdown_engine/scan.js@@ -0,0 +1,213 @@+/* eslint-disable no-undef, no-unused-vars */+// Reads the parsed token tree into compact per-post data: per inline block, the+// construct values the engine recognized plus the block's line map, and+// separately the line maps of code/html/quote blocks. A reference definition+// becomes a block of its own holding the destination its line spells, because+// the links and images that resolve through it spell no URL where they stand.+// Only this compact data crosses the V8 boundary, never the token tree itself.++function __scanCountOccurrences(haystack, needle) {+ // An empty needle would match at every position without ever advancing the+ // cursor (indexOf("") is always the current index).+ if (needle.length === 0) {+ return 0;+ }+ let count = 0;+ let index = haystack.indexOf(needle);+ while (index !== -1) {+ count += 1;+ index = haystack.indexOf(needle, index + needle.length);+ }+ return count;+}++// How often the link's destination also appears in its label text — a+// `[URL](same URL)` self-link writes the value twice in the raw, so count+// matching must expect both occurrences. The schemeless reading covers a label+// spelling the bare domain of a linkified destination.+function __scanLabelHits(label, href) {+ if (href.length === 0) {+ return 0;+ }+ if (label.includes(href)) {+ return __scanCountOccurrences(label, href);+ }+ const bare = href.replace(/^https?:\/\//, "");+ if (bare.length > 0 && bare !== href && label.includes(bare)) {+ return __scanCountOccurrences(label, bare);+ }+ return 0;+}++function __scanWalk(children, block) {+ if (!children) {+ return;+ }+ const linkStack = [];+ for (let i = 0; i < children.length; i++) {+ const child = children[i];+ if (child.type === "mention_open") {+ const next = children[i + 1];+ if (next && next.type === "text") {+ block.mentions.push(next.content);+ }+ } else if (child.type === "link_open") {+ const hashtagType = child.attrGet("data-type");+ // The upload protocol rewrites unresolved short URLs into a placeholder+ // and stores the original in data-orig-*; the original is the construct.+ const href = child.attrGet("data-orig-href") || child.attrGet("href");+ if (hashtagType !== null) {+ // For a hashtag the slug and type are the construct — the href shape+ // depends on lookup internals that scanning replaces.+ block.hashtags.push({+ type: hashtagType,+ slug: child.attrGet("data-ref") || child.attrGet("data-slug") || "",+ });+ linkStack.push(null);+ } else if (child.meta && child.meta.label) {+ // A reference-style `[text][label]` link carries the destination its+ // definition spells, and the raw here holds none of it. The definition+ // line reports that destination itself.+ linkStack.push(null);+ } else if (href !== null && href[0] !== "#") {+ // Fragment-only hrefs are intra-post anchors, never remapped. A+ // linkified or autolinked URL is its own label but exists once in the+ // raw, so only an explicit `[label](dest)` link contributes label+ // occurrences.+ const explicit = child.markup !== "linkify" && child.info !== "auto";+ linkStack.push({ href, label: "", explicit });+ } else {+ linkStack.push(null);+ }+ } else if (child.type === "link_close") {+ const open = linkStack.pop();+ if (open) {+ block.links.push({+ href: open.href,+ labelHits: open.explicit ? __scanLabelHits(open.label, open.href) : 0,+ });+ }+ } else if (child.type === "image") {+ const src = child.attrGet("data-orig-src") || child.attrGet("src");+ // A reference-style `![alt][label]` image spells no URL either, but its+ // alt text can still hold constructs of its own.+ if (src !== null && !(child.meta && child.meta.label)) {+ block.images.push(src);+ }+ __scanWalk(child.children, block);+ } else if (child.type === "emoji") {+ const title = child.attrGet("title");+ if (title !== null) {+ block.emojis.push(title.replace(/^:|:$/g, ""));+ }+ } else if (child.type === "code_inline") {+ block.code += 1;+ } else {+ if (child.type === "text" && linkStack.length > 0) {+ const open = linkStack[linkStack.length - 1];+ if (open) {+ open.label += child.content;+ }+ }+ if (child.children) {+ __scanWalk(child.children, block);+ }+ }+ }+}++// The destination of a reference definition, as a block of its own: the+// definition line is the one place the raw spells that URL. markdown-it keeps+// the first definition of a label and ignores every later one, so a repeated+// label spells a destination the parse resolves nothing through.+function __scanDefinition(token, env, seenLabels) {+ const label = token.meta ? token.meta.label : null;+ if (!label || seenLabels.has(label)) {+ return null;+ }+ seenLabels.add(label);++ // A label colliding with an Object member ("toString") is never stored, and+ // the parse resolves nothing through it either.+ const reference = env.references ? env.references[label] : null;+ if (!reference || typeof reference.href !== "string") {+ return null;+ }++ return {+ map: token.map,+ mentions: [],+ hashtags: [],+ links: [{ href: reference.href, labelHits: 0 }],+ images: [],+ emojis: [],+ code: 0,+ };+}++function __scanOne(post) {+ const env = {};+ const tokens = __pt.parse(post.raw, env);+ const blocks = [];+ const blockTokens = [];+ const seenLabels = new Set();+ let quoteHeader = null;+ for (const token of tokens) {+ if (token.type === "bbcode_open" && token.attrGet("data-username") !== null) {+ quoteHeader = token;+ }+ if (token.type === "inline") {+ const block = {+ map: token.map,+ mentions: [],+ hashtags: [],+ links: [],+ images: [],+ emojis: [],+ code: 0,+ };+ __scanWalk(token.children, block);+ if (+ block.mentions.length ||+ block.hashtags.length ||+ block.links.length ||+ block.images.length ||+ block.emojis.length ||+ block.code > 0+ ) {+ blocks.push(block);+ }+ } else if (token.type === "reference_definition") {+ const definition = __scanDefinition(token, env, seenLabels);+ if (definition) {+ blocks.push(definition);+ }+ } else if (+ token.map &&+ (token.type === "fence" ||+ token.type === "code_block" ||+ token.type === "html_block" ||+ // A quote's outer aside token carries no map; the inner blockquote+ // bbcode token does, so every mapped bbcode block is recorded with its+ // tag.+ token.type === "bbcode_open")+ ) {+ const entry = { type: token.type, tag: token.tag, map: token.map };+ if (token.type === "bbcode_open" && token.tag === "blockquote" && quoteHeader) {+ // The header fields make one quote distinguishable from another, so+ // a substitution check can see its own quote change. They sit on the+ // mapless aside token that opens the quote, not on this one.+ entry.username = quoteHeader.attrGet("data-username");+ entry.post = quoteHeader.attrGet("data-post");+ entry.topic = quoteHeader.attrGet("data-topic");+ quoteHeader = null;+ }+ blockTokens.push(entry);+ }+ }+ return { id: post.id, blocks, blockTokens };+}++function __scanPosts(posts) {+ return posts.map(__scanOne);+}diff --git a/migrations/converters/lib/migrations/converters/markdown_renderer.rb b/migrations/converters/lib/migrations/converters/markdown_renderer.rbindex c3f4a3b56e9..85cf83f85c6 100644--- a/migrations/converters/lib/migrations/converters/markdown_renderer.rb+++ b/migrations/converters/lib/migrations/converters/markdown_renderer.rb@@ -44,7 +44,7 @@ module Migrations# Embeds deferred unless the caller narrows or extends the set. These three# can't be finalized at convert time: an upload must first exist on the- # destination, an attributed quote references a source post the import+ # destination, a quote can point at a source post the import# renumbers, and a mention names a user a merge may rename — they can only be# resolved at import time, using the import maps.#@@ -137,13 +137,13 @@ module Migrationsenddef defer_quote(node, interface)- # Only the attribution carries the foreign post/topic/user reference that+ # Only the opening tag carries the foreign post/topic/user reference that# needs remapping; the quoted body renders normally and the closing tag- # stays in place. A quote attributed only by a display name (`author`) has- # nothing to remap, so it renders natively and keeps that attribution as+ # stays in place. A quote that names only a display name (`author`) has+ # nothing to remap, so it renders natively and keeps that header as# written. The token stands in for the opening `[quote="…"]`, which# PlaceholderResolver#render_quote rebuilds.- return interface.render_default(node) unless attributed?(node)+ return interface.render_default(node) unless remappable?(node)token =@embeds.quote(@@ -185,17 +185,17 @@ module Migrationsend# Whether the quote carries a source post/topic/user reference that needs- # remapping. A quote attributed only by a display name has none.- def attributed?(node)+ # remapping. A quote that names only a display name has none.+ def remappable?(node)node.post_number || node.topic_id || node.post_id || node.user_id || node.usernameend- # Markbridge attribution numbers are unbounded Integers, but the+ # Markbridge quote ids are unbounded Integers, but the# IntermediateDB stores ids as SQLite signed 64-bit integers (binding a# bignum raises). At most 18 digits fit. A longer number is a numeric post- # title or junk, not a real id — drop it and let the remaining attribution- # carry the quote. The Discourse MarkdownScanner quote detector applies the- # same 18-digit bound (see markdown_scanner/detectors/quote.rb).+ # title or junk, not a real id — drop it and let the remaining ids+ # carry the quote. The Discourse MarkdownScanner quote construct applies the+ # same 18-digit bound (see markdown_scanner/constructs/quote.rb).def storable_id(value)value if value && value < 10**18enddiff --git a/migrations/converters/migrations-converters.gemspec b/migrations/converters/migrations-converters.gemspecindex 6f268029dcf..b4cd0bc9ed3 100644--- a/migrations/converters/migrations-converters.gemspec+++ b/migrations/converters/migrations-converters.gemspec@@ -12,8 +12,12 @@ Gem::Specification.new do |s|s.add_dependency "migrations-core"s.add_dependency "activesupport"s.add_dependency "colored2"+ # Both unpinned to follow the host application's Gemfile, which is the+ # authority on the V8 and emoji-data versions the engine context must match.+ s.add_dependency "discourse-emojis"s.add_dependency "i18n"s.add_dependency "markbridge", ">= 0.4.0"+ s.add_dependency "mini_racer"s.add_dependency "pg"s.add_dependency "zeitwerk"enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/engine_scanner_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/engine_scanner_spec.rbnew file mode 100644index 00000000000..f917a114fe0--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/engine_scanner_spec.rb@@ -0,0 +1,1139 @@+# frozen_string_literal: true++# The engine tier end to end: RawExtractor with a real MarkdownEngine::Context,+# so the constructs come out of the actual discourse-markdown-it parse and the+# positions out of count matching. The engine context needs no Rails, so+# this runs in the isolated suite.+RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ let(:custom_emoji_names) { %w[parrot] }+ let(:category_slugs) { %w[support] }+ let(:tag_names) { %w[release] }+ let(:internal_link_hosts) { { "forum.example.com" => nil } }+ let(:refusals) { [] }++ let(:markdown_engine) do+ MarkdownEngineHelper.context_for(+ category_slugs:,+ tag_names:,+ custom_emoji_names:,+ settings: {+ "unicode_usernames" => true,+ },+ )+ end++ let(:on_engine_refusal) { ->(cause, _detail) { refusals << cause } }++ describe "code shielding through the real parse" do+ it "replaces the prose mention and leaves the code-span one alone" do+ output = extract("`@bob` in code and @alice in prose")++ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to start_with("`@bob` in code and ")+ expect(output).to include(buffer.mentions.first[:placeholder])+ end++ it "replaces the prose copy and leaves the fenced copy of the same name" do+ output = extract("@alice wrote:\n\n```\n@alice\n```\n")++ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to include("```\n@alice\n```")+ expect(output).to start_with(buffer.mentions.first[:placeholder])+ end++ it "confirms the prose copy by substitution when the same name also sits in a code span" do+ output = extract("`@alice` and @alice")++ # Count matching cannot split the pair, but the substitution pass confirms+ # the prose occurrence positionally: substituting it removes a mention+ # from the parse, substituting the code-span copy removes nothing.+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to eq("`@alice` and #{buffer.mentions.first[:placeholder]}")+ expect(refusals).to be_empty+ end++ it "confirms every prose copy when several are mixed with a code-span copy" do+ # Three raw occurrences against two engine mentions: count matching+ # refuses, substitution confirms exactly the two prose ones, and every construct+ # the engine reported is then placed — nothing stays on the tally.+ output = extract("@alice, `@alice` and @alice")++ expect(buffer.mentions.size).to eq(2)+ expect(output).to include("`@alice`")+ expect(refusals).to be_empty+ expect(extractor.engine_refusals).to be_empty+ end+ end++ describe "gate agreement" do+ it "defers nothing for a plain quote block, so the gate may skip it" do+ # The mention forces the engine path; the plain [quote] block riding+ # along yields no embed — which is why TierGate's quote candidacy can+ # require the metadata-bearing `[quote=` shape and stay a superset of+ # what extraction defers.+ extract("[quote]\nquoted prose\n[/quote]\n\n@alice")++ expect(buffer.quotes).to be_empty+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ prepared = extractor.prepare(raw: "[quote]\nquoted prose\n[/quote]")+ expect(prepared).not_to be_engine_bound+ end++ it "matches a decomposed hashtag spelling against its composed name" do+ composed = "café"+ engine =+ MarkdownEngineHelper.context_for(+ tag_names: [composed],+ settings: {+ "unicode_usernames" => true,+ },+ )+ extractor =+ described_class.new(+ embeds: buffer,+ mention_names:,+ hashtag_names:+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize(composed)]),+ markdown_engine: engine,+ )++ decomposed = "café"+ output = extractor.extract("tagged #{"#" + decomposed} here")++ expect(buffer.hashtags.size).to eq(1)+ expect(buffer.hashtags.first[:original_markdown]).to eq("##{decomposed}")+ expect(output).to eq("tagged #{buffer.hashtags.first[:placeholder]} here")+ end+ end++ describe "constructs on the engine tier" do+ it "extracts an upload image whole, keeping its alt and dimensions in the row source" do+ raw = "see `code`\n\n"+ output = extract(raw)++ expect(buffer.uploads.size).to eq(1)+ expect(buffer.uploads.first[:original_markdown]).to eq(+ "",+ )+ expect(output).to end_with(buffer.uploads.first[:placeholder])+ end++ it "extracts a titled internal link with its verbatim source" do+ extract(%{read [topic](https://forum.example.com/t/slug/5 "context") `x`})++ expect(buffer.links.size).to eq(1)+ expect(buffer.links.first[:original_markdown]).to eq(+ %{[topic](https://forum.example.com/t/slug/5 "context")},+ )+ end++ it "covers both occurrences of a self-link with one node" do+ raw = "[https://forum.example.com/t/slug/5](https://forum.example.com/t/slug/5) `x`"+ output = extract(raw)++ expect(buffer.links.size).to eq(1)+ expect(output).to start_with(buffer.links.first[:placeholder])+ expect(output).not_to include("forum.example.com")+ end++ it "rewrites a reference-link definition in place" do+ raw = "so [topic][1] `x`\n\n[1]: https://forum.example.com/t/slug/5\n"+ output = extract(raw)++ expect(buffer.links.size).to eq(1)+ expect(output).to include("[topic][1]")+ expect(output).to include("[1]: #{buffer.links.first[:placeholder]}")+ end++ it "remaps the single-line quote form the line-oriented walks cannot take" do+ raw = %{[quote="alice, post:2, topic:5"]inline body[/quote]}+ output = extract(raw)++ expect(buffer.quotes.first).to include(+ quoted_username: "alice",+ quoted_post_number: 2,+ quoted_topic_id: 5,+ original_markdown: %{[quote="alice, post:2, topic:5"]},+ )+ expect(output).to end_with("inline body[/quote]")+ end++ it "extracts a quote opener and the mention inside its body" do+ output = extract(%{`x`\n[quote="bob, post:1, topic:2"]\nthanks @alice\n[/quote]})++ expect(buffer.quotes.first).to include(quoted_username: "bob")+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to include("[/quote]")+ end+ end++ describe "fail-closed behavior" do+ it "extracts an angle-bracket destination whole" do+ extract("[x](<https://forum.example.com/t/slug/5>) `y`")++ expect(buffer.links.first).to include(+ url: "https://forum.example.com/t/slug/5",+ original_markdown: "[x](<https://forum.example.com/t/slug/5>)",+ )+ end++ it "refuses a recognized reference no grammar can take, instead of calling it handled" do+ # An autolinked `<upload://…>` is a real link to the engine, but no+ # construct grammar has a form for it and it is neither a destination+ # nor a definition. The reference stays stale — that must land on the+ # must-resolve tally, never report as success. The mention beside it is+ # still extracted.+ raw = "@alice sees <upload://#{sha1}.png> `y`"+ output = extract(raw)++ expect(buffer.uploads).to be_empty+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to include("<upload://#{sha1}.png>")+ expect(refusals).to eq(%i[unanchored])+ end++ it "rewrites the destination of an upload link whose label no grammar can take" do+ # CommonMark allows an escaped `]` inside a label; the construct grammar+ # does not. The destination is still the upload it is, and the label+ # keeps its own bytes, so nothing around it is lost.+ raw = "sees [a \\] b](upload://#{sha1}.png) `y`"+ output = extract(raw)++ expect(buffer.uploads.first).to include(original_markdown: "upload://#{sha1}.png")+ expect(output).to eq("sees [a \\] b](#{buffer.uploads.first[:placeholder]}) `y`")+ expect(refusals).to be_empty+ end++ it "rewrites the destination of a link whose label no grammar can take" do+ raw = "sees [a \\] b](https://forum.example.com/t/slug/5) `y`"+ output = extract(raw)++ expect(buffer.links.first).to include(+ url: "https://forum.example.com/t/slug/5",+ text: nil,+ original_markdown: "https://forum.example.com/t/slug/5",+ url_offset: 0,+ )+ expect(output).to eq("sees [a \\] b](#{buffer.links.first[:placeholder]}) `y`")+ expect(refusals).to be_empty+ end++ it "leaves an external link untouched" do+ raw = "`x` [docs](https://elsewhere.example.org/page)"+ output = extract(raw)++ expect(output).to eq(raw)+ expect(buffer.links).to be_empty+ expect(refusals).to be_empty+ end++ it "extracts the literal mention and keeps the entity-spelled one on the tally" do+ output = extract("@alice and @bob `x`")++ # The engine decodes `@bob` into a mention no literal byte sequence+ # spells, so no substitution check can confirm it — it stays verbatim and keeps the+ # body's cause. The literal mention next to it is confirmed by substitution and+ # extracted anyway.+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to include("@bob")+ expect(output).to start_with(buffer.mentions.first[:placeholder])+ expect(refusals).to eq(%i[entity])+ end++ it "extracts from CR line endings through the map-free substitution pass" do+ output = extract("@alice\r\nhello `x`")++ # Count matching cannot index a CR body (markdown-it normalizes CR+ # away before its line maps), but the substitution check's token delta needs no+ # maps at all.+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to eq("#{buffer.mentions.first[:placeholder]}\r\nhello `x`")+ expect(refusals).to be_empty+ end+ end++ describe "soundness regressions" do+ it "rewrites the prose spelling, not an equivalent spelling inside code" do+ # The engine links the schemeless prose URL and normalizes its href to+ # the scheme-ful form — which happens to sit inside the code span. A+ # per-reading count would match the code span and corrupt it; the+ # span union counts both spellings and the substitution check confirms only the prose+ # one is live.+ raw = "`http://forum.example.com/t/slug/5` and forum.example.com/t/slug/5"+ output = extract(raw)++ expect(buffer.links.size).to eq(1)+ expect(buffer.links.first).to include(+ url: "forum.example.com/t/slug/5",+ target_id: 5,+ original_markdown: "forum.example.com/t/slug/5",+ )+ expect(output).to start_with("`http://forum.example.com/t/slug/5` and ")+ expect(output).to end_with(buffer.links.first[:placeholder])+ expect(refusals).to be_empty+ end++ # markdown-it strips a backslash escape before the text rules run, so core+ # cooks `\\@bob` as a mention. A counter that read the escape as a+ # rejection counted one occurrence too few here and handed the token to the+ # code-span look-alike — the post came back with its code span rewritten.+ [+ ["mention", "\\@bob", "`@bob`", :mentions, "bob"],+ ["hashtag", "\\#support", "`#support`", :hashtags, "support"],+ ["custom emoji", "\\:parrot:", "`:parrot:`", :emojis, "parrot"],+ ].each do |kind, escaped, shielded, rows, name|+ it "rewrites an escaped #{kind} and leaves the code-span copy alone" do+ output = extract("#{escaped} and #{shielded}")++ expect(buffer.public_send(rows).map { |row| row[:name] }).to eq([name])+ placeholder = buffer.public_send(rows).first[:placeholder]+ expect(output).to eq("\\#{placeholder} and #{shielded}")+ expect(refusals).to be_empty+ end+ end++ it "rewrites a shortcode the author upper-cased and leaves the code-span copy alone" do+ # Core lowercases a shortcode before it looks a custom emoji up, so+ # `:PARROT:` is the same emoji — and counting has to fold the raw the+ # same way, or the engine's one token lands on the code span.+ output = extract(":PARROT: and `:parrot:`")++ expect(buffer.emojis.map { |row| row[:name] }).to eq(%w[PARROT])+ expect(output).to eq("#{buffer.emojis.first[:placeholder]} and `:parrot:`")+ expect(refusals).to be_empty+ end++ it "records a shortcode in the author's own case" do+ output = extract("nice :ParRot: work")++ expect(buffer.emojis.map { |row| row[:name] }).to eq(%w[ParRot])+ expect(output).to eq("nice #{buffer.emojis.first[:placeholder]} work")+ expect(refusals).to be_empty+ end++ it "stores the raw spelling of a swallowed query, not the href's percent-encoding" do+ # Linkify takes the quote and everything after it into the href and+ # percent-encodes it there. The href may only resolve the host: the+ # suffix the importer writes back is read from the raw bytes, so the+ # author's spelling survives the origin rewrite.+ raw = %{key: https://forum.example.com/user-api-key/new?public_key="AAAAB3NzaC1yc2E" ok}+ output = extract(raw)++ expect(buffer.links.size).to eq(1)+ expect(buffer.links.first).to include(+ target_suffix: %{/user-api-key/new?public_key="AAAAB3NzaC1yc2E"},+ original_markdown:+ %{https://forum.example.com/user-api-key/new?public_key="AAAAB3NzaC1yc2E"},+ )+ expect(output).to eq("key: #{buffer.links.first[:placeholder]} ok")+ expect(refusals).to be_empty+ end++ it "stores a typed target's suffix in the raw spelling too" do+ # Linkify swallows a balanced quote pair into the href and encodes it;+ # the suffix stays in the author's spelling.+ raw = %{see https://forum.example.com/t/slug/5?q="abc" now}+ output = extract(raw)++ expect(buffer.links.size).to eq(1)+ expect(buffer.links.first).to include(target_id: 5, target_suffix: %{?q="abc"})+ expect(output).to eq("see #{buffer.links.first[:placeholder]} now")+ expect(refusals).to be_empty+ end++ it "rewrites a relative reference definition" do+ raw = "see [old topic][1] `x`\n\n[1]: /t/slug/5\n"+ output = extract(raw)++ expect(buffer.links.size).to eq(1)+ expect(buffer.links.first).to include(url: "/t/slug/5", target_id: 5)+ expect(output).to include("[1]: #{buffer.links.first[:placeholder]}")+ expect(refusals).to be_empty+ end++ it "rewrites one reference definition that several links share" do+ # The uses spell no URL of their own; the definition line does, and+ # rewriting it carries both links along, so one row is enough.+ raw =+ "see [profile][prefs] and [settings][prefs] `x`\n\n" \+ "[prefs]: https://forum.example.com/t/slug/5\n"+ output = extract(raw)++ expect(buffer.links.size).to eq(1)+ expect(buffer.links.first).to include(url: "https://forum.example.com/t/slug/5", target_id: 5)+ expect(output).to include("[prefs]: #{buffer.links.first[:placeholder]}")+ expect(refusals).to be_empty+ end++ it "rewrites several distinct shared definitions in one body" do+ raw =+ "see [a][1], [b][1], [c][2] and [d][2] `x`\n\n" \+ "[1]: https://forum.example.com/t/slug/5\n[2]: https://forum.example.com/t/slug/7\n"+ output = extract(raw)++ expect(buffer.links.map { |row| row[:target_id] }).to contain_exactly(5, 7)+ buffer.links.each { |row| expect(output).to include(row[:placeholder]) }+ expect(refusals).to be_empty+ end++ it "rewrites a shared definition but not a copy of its URL inside code" do+ # The definition is counted in its own line region, where it is the only+ # occurrence. The code-span copy sits outside that region, and no region+ # expects it, so nothing touches it.+ raw =+ "see [a][1] and [b][1] and `https://forum.example.com/t/slug/5`\n\n" \+ "[1]: https://forum.example.com/t/slug/5\n"+ output = extract(raw)++ expect(buffer.links.size).to eq(1)+ expect(output).to include("`https://forum.example.com/t/slug/5`")+ expect(output).to include("[1]: #{buffer.links.first[:placeholder]}")+ expect(refusals).to be_empty+ end++ it "rewrites a shared definition but not a definition-shaped line inside a fence" do+ # Two raw occurrences, both shaped like a definition. Only the live one+ # produces a token, and it is counted inside its own line region; the+ # fenced copy lies in no region with a count of its own.+ fenced_line = "[also]: https://forum.example.com/t/slug/5"+ raw =+ "see [a][1] and [b][1]\n\n" \+ "[1]: https://forum.example.com/t/slug/5\n\n" \+ "```text\n#{fenced_line}\n```\n"+ output = extract(raw)++ expect(buffer.links.size).to eq(1)+ expect(output).to include(fenced_line)+ expect(output).to include("[1]: #{buffer.links.first[:placeholder]}")+ expect(refusals).to be_empty+ end++ it "keeps a definition-shaped line inside a fence that defines nothing" do+ # Nothing outside the fence spells the URL, so no region expects it and+ # the fenced line is left exactly as the author wrote it.+ fenced_line = "[1]: https://forum.example.com/t/slug/5"+ raw = "see `x`\n\n```text\n#{fenced_line}\n```\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ expect(refusals).to be_empty+ end++ it "rewrites the first definition of a repeated label and leaves the second verbatim" do+ # Every use resolves through the first definition of a label; the second+ # spells a URL the parse never reaches, so it is no occurrence of+ # anything and keeps its own bytes.+ shadowed = "https://forum.example.com/t/slug/7"+ raw = "see [x][1] `y`\n\n[1]: https://forum.example.com/t/slug/5\n[1]: #{shadowed}\n"+ output = extract(raw)++ expect(buffer.links.size).to eq(1)+ expect(buffer.links.first).to include(target_id: 5)+ expect(output).to include("[1]: #{buffer.links.first[:placeholder]}")+ expect(output).to include("[1]: #{shadowed}")+ expect(refusals).to be_empty+ end++ it "resolves a shared definition without a substitution check" do+ # The definition's line region expects one occurrence and holds one, so+ # the body is placed by the counting pass alone — the single parse.+ parses = []+ raw =+ "see [profile][prefs] and [settings][prefs] `x`\n\n" \+ "[prefs]: https://forum.example.com/t/slug/5\n"+ output = extractor_recording_parses(parses).extract(raw)++ expect(parses.size).to eq(1)+ expect(buffer.links.size).to eq(1)+ expect(output).to include("[prefs]: #{buffer.links.first[:placeholder]}")+ expect(refusals).to be_empty+ end++ # One parse means the body matched by counting; every further parse is a+ # substitution check.+ def extractor_recording_parses(parses)+ engine = instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)+ allow(engine).to receive(:scan) do |posts, timeout_ms: nil|+ parses << posts+ markdown_engine.scan(posts)+ end+ described_class.new(+ embeds: buffer,+ mention_names:,+ hashtag_names:,+ markdown_engine: engine,+ internal_link_hosts:,+ on_engine_refusal:,+ )+ end++ it "counts a root link once when longer links on the same host follow it" do+ # The root link's schemeless reading also sits in every `/u/…` link on the+ # host, so a digest post used to escalate once per user it names.+ parses = []+ raw =+ "welcome to https://forum.example.com/\n\n" \+ "[bob](https://forum.example.com/u/bob) and [carol](https://forum.example.com/u/carol)"+ output = extractor_recording_parses(parses).extract(raw)++ expect(parses.size).to eq(1)+ expect(buffer.links.size).to eq(3)+ buffer.links.each { |row| expect(output).to include(row[:placeholder]) }+ expect(refusals).to be_empty+ end++ it "still escalates when a root link's own spelling sits in a code span" do+ parses = []+ raw =+ "`https://forum.example.com/` and https://forum.example.com/ " \+ "and [bob](https://forum.example.com/u/bob)"+ output = extractor_recording_parses(parses).extract(raw)++ expect(parses.size).to be > 1+ expect(buffer.links.size).to eq(2)+ expect(output).to start_with("`https://forum.example.com/` and ")+ expect(refusals).to be_empty+ end++ it "counts a post link that another tracked link spells as its prefix" do+ # A table of contents links post 3 and post 30 of the same topic: the+ # shorter destination occurs inside the longer one, and every such pair+ # used to cost the body its substitution budget.+ parses = []+ raw =+ "1. [three](https://forum.example.com/t/slug/5/3)\n" \+ "2. [thirty](https://forum.example.com/t/slug/5/30)\n"+ output = extractor_recording_parses(parses).extract(raw)++ expect(parses.size).to eq(1)+ expect(buffer.links.map { |row| row[:target_post_number] }).to contain_exactly(3, 30)+ buffer.links.each { |row| expect(output).to include(row[:placeholder]) }+ expect(refusals).to be_empty+ end++ it "confirms the prefix link when the longer one also sits in a code span" do+ # The code copy shares the block with the live links, so the longer value+ # counts two against one and the body escalates — and the shorter value's+ # hits inside both copies must stay dropped, or a confirmed span would+ # cover the code span.+ code = "`https://forum.example.com/t/slug/5/30`"+ parses = []+ raw =+ "see [three](https://forum.example.com/t/slug/5/3) and " \+ "[thirty](https://forum.example.com/t/slug/5/30) and #{code}\n"+ output = extractor_recording_parses(parses).extract(raw)++ expect(parses.size).to be > 1+ expect(buffer.links.map { |row| row[:target_post_number] }).to contain_exactly(3, 30)+ expect(output).to include(code)+ expect(refusals).to be_empty+ end++ it "matches a destination the engine escaped further than the author wrote it" do+ # markdown-it escapes the quotes of the destination and keeps the author's+ # own `%20`, so its href occurs nowhere in the raw. Each escape is read as+ # either spelling, one character at a time.+ raw = %{see <https://forum.example.com/t/slug/5?q="%20a%20b"> now}+ output = extract(raw)++ expect(buffer.links.size).to eq(1)+ expect(buffer.links.first).to include(target_id: 5, target_suffix: %{?q="%20a%20b"})+ expect(output).to eq("see <#{buffer.links.first[:placeholder]}> now")+ expect(refusals).to be_empty+ end++ it "refuses a body with more distinct tracked URLs than the value cap" do+ count =+ Migrations::Converters::Discourse::MarkdownScanner::EngineScanner::MAX_SCANNED_VALUES + 1+ raw = (1..count).map { |i| "https://forum.example.com/t/s/#{i}" }.join(" ")+ output = extract(raw)++ expect(output).to eq(raw)+ expect(buffer.links).to be_empty+ expect(refusals).to eq(%i[url_volume])+ end++ it "extracts both mentions when one name is a prefix of the other" do+ # `@bob` occurs twice in the raw — once on its own, once inside `@bobby` —+ # against the engine's one token, so counting escalates. Substitution+ # then confirms the standalone one and rejects the spelling inside the+ # longer name, which is what keeps both mentions and their spans right.+ extractor =+ described_class.new(+ embeds: buffer,+ mention_names:+ Migrations::CompactStringSet.new(+ %w[bob bobby].map { |name| Migrations::NameNormalizer.normalize(name) },+ ),+ hashtag_names:,+ markdown_engine:,+ on_engine_refusal:,+ )++ output = extractor.extract("@bob and @bobby")++ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[bob bobby])+ placeholders = buffer.mentions.map { |row| row[:placeholder] }+ expect(output).to eq("#{placeholders[0]} and #{placeholders[1]}")+ expect(refusals).to be_empty+ end++ it "refuses a body with more distinct tracked names than the value cap" do+ # Locating a name costs one scan of the body, exactly as a URL does.+ count =+ Migrations::Converters::Discourse::MarkdownScanner::EngineScanner::MAX_SCANNED_VALUES + 1+ names = (1..count).map { |i| format("user%04d", i) }+ raw = names.map { |name| "@#{name}" }.join(" ")+ extractor =+ described_class.new(+ embeds: buffer,+ mention_names:+ Migrations::CompactStringSet.new(+ names.map { |name| Migrations::NameNormalizer.normalize(name) },+ ),+ hashtag_names:,+ markdown_engine:,+ on_engine_refusal:,+ )++ expect(extractor.extract(raw)).to eq(raw)+ expect(buffer.mentions).to be_empty+ expect(refusals).to eq(%i[name_volume])+ end++ it "reports quote headers beyond the substitution limit instead of calling them handled" do+ # CR endings force every quote onto the substitution path; two more quotes+ # than MAX_SUBSTITUTIONS leaves two headers stale, which must show up on the+ # tally rather than reading as a fully handled body.+ max =+ Migrations::Converters::Discourse::MarkdownScanner::EngineScanner::SubstitutionPass::MAX_SUBSTITUTIONS+ quotes =+ (1..(max + 2)).map { |i| "[quote=\"alice, post:#{i}, topic:9\"]\r\nbody #{i}\r\n[/quote]" }+ output = extract(quotes.join("\r\n"))++ expect(buffer.quotes.size).to eq(max)+ expect(output.scan(/\[quote=/).size).to eq(2)+ expect(refusals).to eq(%i[substitution_limit])+ end++ it "stops enumerating a repeated value's occurrences at the substitution limit" do+ engine_calls = 0+ counting_engine =+ instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)+ allow(counting_engine).to receive(:scan) do |posts, timeout_ms: nil|+ engine_calls += 1+ markdown_engine.scan(posts)+ end+ extractor =+ described_class.new(+ embeds: buffer,+ mention_names:,+ hashtag_names:,+ markdown_engine: counting_engine,+ on_engine_refusal:,+ )+ max =+ Migrations::Converters::Discourse::MarkdownScanner::EngineScanner::SubstitutionPass::MAX_SUBSTITUTIONS++ # 200 live mentions plus one code-span copy: counting refuses the body,+ # and the substitution pass pays at most the limit — the code-span copy+ # burns the first check, the confirmed tail ends at the limit.+ output = extractor.extract("`@alice` #{(["@alice"] * 200).join(" ")}")++ # One initial parse, then one parse per check up to the limit; the+ # occurrences past the limit cost no engine call.+ expect(engine_calls).to eq(1 + max)+ expect(buffer.mentions.size).to eq(max - 1)+ expect(output).to start_with("`@alice` ")+ expect(refusals).to eq(%i[substitution_limit])+ end++ it "keeps a non-default port as part of the host identity, scheme-aware" do+ raw = [+ "on `x`:",+ "https://forum.example.com:80/t/slug/5",+ "http://forum.example.com:443/t/slug/5",+ "//forum.example.com:8443/t/slug/5 done",+ ].join("\n")+ output = extract(raw)++ # `:80` is only default for http and `:443` only for https; a+ # protocol-relative URL has no scheme to default from. None of these+ # origins is the configured host, so none is a construct — and none is+ # a refusal either, external links never are.+ expect(buffer.links).to be_empty+ expect(output).to eq(raw)+ expect(refusals).to be_empty+ end++ context "with a subdirectory install" do+ let(:internal_link_hosts) { { "forum.example.com" => "/forum" } }++ it "tracks only paths inside the host's prefix" do+ raw =+ "`x` [a](https://forum.example.com/other/page) " \+ "and https://forum.example.com/forum/t/slug/5"+ output = extract(raw)++ # The sibling app's path is not the forum's; it must neither be+ # rewritten nor refused, while the in-prefix URL resolves normally.+ expect(buffer.links.size).to eq(1)+ expect(buffer.links.first).to include(target_id: 5)+ expect(output).to include("[a](https://forum.example.com/other/page)")+ expect(refusals).to be_empty+ end+ end++ it "scrubs invalid bytes instead of raising, once, at the top" do+ raw = "\xFF @alice `x`".dup.force_encoding(Encoding::UTF_8)+ output = extract(raw)++ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to start_with("� ")+ expect(output).to include(buffer.mentions.first[:placeholder])+ end++ it "records the destination span so resolution never touches a title" do+ extract(+ %{[docs](https://forum.example.com/t/slug/5 "see https://forum.example.com/t/slug/5") `x`},+ )++ row = buffer.links.first+ # The span points at the destination; the same URL inside the title is+ # outside it, so the importer's span rewrite cannot reach it.+ expect(row[:url_offset]).to eq("[docs](".bytesize)+ expect(row[:label_url_offset]).to be_nil+ end++ it "records the label span of a self-link" do+ extract("[https://forum.example.com/t/slug/5](https://forum.example.com/t/slug/5) `x`")++ row = buffer.links.first+ expect(row[:label_url_offset]).to eq(1)+ expect(row[:url_offset]).to eq("[https://forum.example.com/t/slug/5](".bytesize)+ end++ it "costs a per-input engine failure one body, not the conversion" do+ failing_engine = instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)+ allow(failing_engine).to receive(:scan).and_raise(+ MiniRacer::ScriptTerminatedError,+ "JavaScript was terminated",+ )+ details = []+ extractor =+ described_class.new(+ embeds: buffer,+ mention_names:,+ hashtag_names:,+ markdown_engine: failing_engine,+ on_engine_refusal:+ lambda do |cause, detail|+ refusals << cause+ details << detail+ end,+ )++ raw = "@alice `x`"+ expect(extractor.extract(raw)).to eq(raw)+ expect(refusals).to eq(%i[engine_error])+ expect(details).to eq(%w[MiniRacer::ScriptTerminatedError])+ # The failed fast attempt plus the failed slow retry, each followed by a+ # context reset so the next body gets a healthy engine.+ expect(failing_engine).to have_received(:scan).twice+ expect(failing_engine).to have_received(:reset!).twice+ expect(extractor.slow_parses).to eq(0)+ end++ it "recovers a body whose parse only succeeds on the slow retry" do+ ceilings = []+ retrying_engine =+ instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)+ allow(retrying_engine).to receive(:scan) do |posts, timeout_ms: nil|+ ceilings << timeout_ms+ raise MiniRacer::ScriptTerminatedError, "terminated" if ceilings.size == 1+ markdown_engine.scan(posts)+ end+ slow_parses_seen = 0+ extractor =+ described_class.new(+ embeds: buffer,+ mention_names:,+ hashtag_names:,+ markdown_engine: retrying_engine,+ on_engine_refusal:,+ on_slow_parse: -> { slow_parses_seen += 1 },+ )++ output = extractor.extract("@alice `x`")++ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to include(buffer.mentions.first[:placeholder])+ expect(refusals).to be_empty+ # The fast attempt carries no override; the retry's ceiling is the time+ # left until the per-body deadline, computed just after the deadline is+ # set, so it comes in at most a moment under the constant.+ slow_ms = Migrations::Converters::Discourse::MarkdownScanner::EngineScanner::SLOW_TIMEOUT_MS+ expect(ceilings.size).to eq(2)+ expect(ceilings.first).to be_nil+ expect(ceilings.last).to be_between(slow_ms - 1_000, slow_ms)+ expect(retrying_engine).to have_received(:reset!).once+ expect(extractor.slow_parses).to eq(1)+ expect(slow_parses_seen).to eq(1)+ end++ it "refuses directly when the slow retry is disabled" do+ failing_engine = instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)+ allow(failing_engine).to receive(:scan).and_raise(+ MiniRacer::ScriptTerminatedError,+ "terminated",+ )+ extractor =+ described_class.new(+ embeds: buffer,+ mention_names:,+ hashtag_names:,+ markdown_engine: failing_engine,+ slow_timeout_ms: nil,+ on_engine_refusal:,+ )++ expect(extractor.extract("@alice `x`")).to eq("@alice `x`")+ expect(refusals).to eq(%i[engine_error])+ expect(failing_engine).to have_received(:scan).once+ end++ it "runs a slow-path body's substitution checks under what is left of the retry deadline" do+ engine_scanner = Migrations::Converters::Discourse::MarkdownScanner::EngineScanner+ budgets = []+ allow(engine_scanner::SubstitutionPass).to receive(+ :new,+ ).and_wrap_original do |original, *args, **kwargs|+ budgets << kwargs[:seconds_budget]+ original.call(*args, **kwargs)+ end+ calls = 0+ retrying_engine =+ instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)+ allow(retrying_engine).to receive(:scan) do |posts, timeout_ms: nil|+ calls += 1+ raise MiniRacer::ScriptTerminatedError, "terminated" if calls == 1+ markdown_engine.scan(posts)+ end+ extractor =+ described_class.new(+ embeds: buffer,+ mention_names:,+ hashtag_names:,+ markdown_engine: retrying_engine,+ on_engine_refusal:,+ )++ # Count matching cannot split this pair, so the body needs its check —+ # which must run with more than the default budget (a single legitimate+ # slow parse may already take longer than that), but only with what the+ # slow parse left of the 30-second retry deadline, never a fresh+ # 30 seconds on top of it.+ output = extractor.extract("`@alice` and @alice")++ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to eq("`@alice` and #{buffer.mentions.first[:placeholder]}")+ expect(refusals).to be_empty+ expect(extractor.slow_parses).to eq(1)+ slow_ceiling = engine_scanner::SLOW_TIMEOUT_MS / 1000.0+ expect(budgets).to all(be > engine_scanner::SubstitutionPass::SUBSTITUTION_SECONDS_BUDGET)+ expect(budgets).to all(be < slow_ceiling)+ end++ it "shrinks every engine call's ceiling toward the deadline and stops at zero" do+ # A scripted clock: each engine call costs seven scripted seconds, so+ # the deadline runs out in the middle of the substitution checks.+ now = 0.0+ allow(Process).to receive(:clock_gettime).and_call_original+ allow(Process).to receive(:clock_gettime).with(Process::CLOCK_MONOTONIC) { now }+ ceilings = []+ calls = 0+ slow_engine = instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)+ allow(slow_engine).to receive(:scan) do |posts, timeout_ms: nil|+ ceilings << timeout_ms+ calls += 1+ now += 7.0+ raise MiniRacer::ScriptTerminatedError, "terminated" if calls == 1+ markdown_engine.scan(posts)+ end+ extractor =+ described_class.new(+ embeds: buffer,+ mention_names:,+ hashtag_names:,+ markdown_engine: slow_engine,+ on_engine_refusal:,+ )++ # One copy in code and five in prose: counts cannot match, so every+ # occurrence needs its own substitution parse — more parses than the+ # deadline has time for.+ output = extractor.extract("`@alice` and @alice and @alice and @alice and @alice and @alice")++ # The fast attempt has no ceiling; the retry's base parse gets the full+ # deadline; each substitution parse gets exactly the time left until+ # the deadline, so no call can run past it, and a call with no time+ # left is declined.+ expect(ceilings).to eq([nil, 30_000, 23_000, 16_000, 9_000, 2_000])+ expect(refusals).to eq(%i[substitution_budget])+ # The code copy was checked first and found to be no construct; three+ # prose occurrences were confirmed before the deadline ran out, the+ # other two stay as written.+ expect(output).to include("`@alice`")+ expect(buffer.mentions.size).to eq(3)+ expect(output.scan("@alice").size).to eq(3)+ end+ end++ describe "prepared batch scanning" do+ let(:body) { "@alice wrote `@bob` and [docs](https://forum.example.com/t/slug/5)" }++ it "normalizes and classifies a body once for a batching caller" do+ invalid = (+"\xFF @alice").force_encoding(Encoding::UTF_8)++ candidate = extractor.prepare(id: 1, raw: invalid, topic_id: 4)+ plain = extractor.prepare(id: 2, raw: "plain text, nothing to find")+ empty = extractor.prepare(id: 3, raw: nil)++ expect(candidate.raw).to be_valid_encoding+ expect(candidate.raw).to eq("� @alice")+ expect(candidate.topic_id).to eq(4)+ expect(candidate).to be_engine_bound+ expect(plain).not_to be_engine_bound+ expect(empty).not_to be_engine_bound+ end++ it "extracts prepared bytes with their precomputed scan data" do+ prepared = extractor.prepare(id: 1, raw: body)+ scan_data = extractor.scan_batches([prepared])+ output = extractor.extract_prepared(prepared, scan_data: scan_data[1])++ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(buffer.links.map { |row| row[:target_id] }).to eq([5])+ expect(output).to include("`@bob`")+ expect(output).to include(buffer.mentions.first[:placeholder])+ end++ it "scans and extracts the same normalized bytes" do+ invalid = (+"\xFF @alice").force_encoding(Encoding::UTF_8)+ prepared = extractor.prepare(id: 1, raw: invalid)+ scan_data = extractor.scan_batches([prepared])++ output = extractor.extract_prepared(prepared, scan_data: scan_data[1])++ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to start_with("� ")+ expect(output).to include(buffer.mentions.first[:placeholder])+ end++ it "returns scan data keyed by id and omits bodies that do not need the engine" do+ candidate = extractor.prepare(id: 7, raw: body)+ plain = extractor.prepare(id: 9, raw: "plain text")+ data = extractor.scan_batches([candidate, plain])++ expect(data.keys).to contain_exactly(7)+ output = extractor.extract_prepared(candidate, scan_data: data[7])+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to include(buffer.mentions.first[:placeholder])+ end++ it "rejects engine-bound bodies with duplicate ids" do+ first = extractor.prepare(id: 7, raw: body)+ second = extractor.prepare(id: 7, raw: "hi @alice")++ # Keying by id would silently keep only one of the results.+ expect { extractor.scan_batches([first, second]) }.to raise_error(ArgumentError, /unique ids/)+ end++ it "bounds each engine call by post count and aggregate bytes" do+ bounded_engine = instance_double(Migrations::Converters::MarkdownEngine::Context)+ allow(bounded_engine).to receive(:scan) do |posts, timeout_ms: nil|+ raise "oversized scan" if posts.size > 2 || posts.sum { |post| post[:raw].bytesize } > 80++ markdown_engine.scan(posts, timeout_ms:)+ end+ allow(bounded_engine).to receive(:reset!)+ batching_extractor =+ described_class.new(+ embeds: buffer,+ mention_names:,+ hashtag_names:,+ custom_emoji_names:,+ internal_link_hosts:,+ markdown_engine: bounded_engine,+ )+ prepared = [+ batching_extractor.prepare(id: 7, raw: body),+ batching_extractor.prepare(id: 8, raw: "hi @alice"),+ batching_extractor.prepare(id: 9, raw: "bye @alice"),+ batching_extractor.prepare(id: 10, raw: "cc @alice"),+ ]++ data = batching_extractor.scan_batches(prepared, max_posts: 2, max_bytes: 80)++ expect(data.keys).to contain_exactly(7, 8, 9, 10)+ end++ it "scans a single body that exceeds the batch byte target" do+ prepared = extractor.prepare(id: 7, raw: "@alice #{"x" * 100}")++ data = extractor.scan_batches([prepared], max_bytes: 10)++ expect(data.keys).to contain_exactly(7)+ end++ it "keeps successful partitions when another partition terminates" do+ flaky_engine = instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)+ allow(flaky_engine).to receive(:scan) do |posts, timeout_ms: nil|+ raise MiniRacer::ScriptTerminatedError if posts.any? { |post| post[:id] == 7 }++ markdown_engine.scan(posts, timeout_ms:)+ end+ batching_extractor =+ described_class.new(+ embeds: buffer,+ mention_names:,+ hashtag_names:,+ custom_emoji_names:,+ internal_link_hosts:,+ markdown_engine: flaky_engine,+ )++ failed = batching_extractor.prepare(id: 7, raw: body)+ successful = batching_extractor.prepare(id: 9, raw: "hi @alice")+ data = batching_extractor.scan_batches([failed, successful], max_posts: 1)++ expect(data.keys).to contain_exactly(9)+ expect(flaky_engine).to have_received(:reset!)+ output = batching_extractor.extract_prepared(failed, scan_data: data[7])+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to include(buffer.mentions.first[:placeholder])+ end+ end++ describe "plugin syntax shielding" do+ context "with the plugin settings enabled on the source" do+ # A source with these plugins enabled parses their blocks with the real+ # plugin rules, so candidate text inside plugin syntax is not prose and+ # must stay as written; the same text in prose beside it is extracted.+ let(:markdown_engine) do+ MarkdownEngineHelper.context_for(+ settings: {+ "unicode_usernames" => true,+ "discourse_events_enabled" => true,+ "discourse_post_event_enabled" => true,+ "discourse_graphviz_enabled" => true,+ "discourse_math_enabled" => true,+ "policy_enabled" => true,+ },+ )+ end++ it "keeps a mention inside an event header and extracts the prose copy" do+ raw = %{[event name="@alice" start="2026-01-01"]\n[/event]\n\nthanks @alice}+ output = extract(raw)++ expect(refusals).to be_empty+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to include(%{[event name="@alice" start="2026-01-01"]})+ expect(output).to end_with("thanks #{buffer.mentions.first[:placeholder]}")+ end++ it "keeps a mention inside a math block" do+ output = extract("$$\n@alice + x\n$$\n\n@alice")++ expect(refusals).to be_empty+ expect(buffer.mentions.size).to eq(1)+ expect(output).to include("$$\n@alice + x\n$$")+ expect(output).to end_with(buffer.mentions.first[:placeholder])+ end++ it "keeps a mention inside a graphviz program" do+ output = extract("[graphviz]\n@alice -> x\n[/graphviz]\n\n@alice")++ expect(refusals).to be_empty+ expect(buffer.mentions.size).to eq(1)+ expect(output).to include("[graphviz]\n@alice -> x\n[/graphviz]")+ expect(output).to end_with(buffer.mentions.first[:placeholder])+ end++ it "keeps a mention inside a policy header while its body stays prose" do+ raw = %{[policy group="@alice"]\nplease agree, @alice\n[/policy]}+ output = extract(raw)++ expect(refusals).to be_empty+ expect(buffer.mentions.size).to eq(1)+ expect(output).to include(%{[policy group="@alice"]})+ expect(output).to include("please agree, #{buffer.mentions.first[:placeholder]}")+ end+ end++ it "treats plugin syntax as prose when the source did not enable the plugin" do+ # The default engine runs without these plugin settings, so `$$` is+ # ordinary text there and the mention in between is a real mention —+ # exactly what such a post cooks to on a site without the plugin.+ output = extractor.extract("$$ @alice $$")++ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])+ expect(output).to eq("$$ #{buffer.mentions.first[:placeholder]} $$")+ end+ end++ describe "upload markers" do+ it "confirms an upload image whose alt text is a URL when a code span forces substitution" do+ short = "upload://qNuWIDpKrjDez7zGdN66jhn5JHW.png"+ body = " and `#{short}`"++ result = extract(body)++ expect(extractor.engine_refusals).to be_empty+ expect(buffer.uploads.size).to eq(1)+ expect(result).to eq("#{buffer.uploads.first[:placeholder]} and `#{short}`")+ end+ end++ describe "substitution markers keep the construct's shape" do+ it "confirms a quote whose body is indented code" do+ # Without the quote, the indented body would become a code block of its+ # own; the marker keeps the quote so only its header changes.+ quote = "[quote=\"sam, post:6, topic:23077\"]\n cd /var/discourse\n[/quote]"+ result = extract("#{quote}\n\n@alice and `@alice`", topic_id: 1)++ expect(extractor.engine_refusals).to be_empty+ expect(buffer.quotes.size).to eq(1)+ expect(buffer.quotes.first).to include(quoted_post_number: 6, quoted_topic_id: 23_077)+ expect(result).to start_with(+ "#{buffer.quotes.first[:placeholder]}\n cd /var/discourse\n[/quote]",+ )+ end++ it "leaves an unclosed opener holding a mention alone without refusing" do+ # Core makes no quote of an opener without a closer; the `@alice` in its+ # header is a live mention, and the opener itself is just text.+ body = "[quote=\"@alice\"]\nno closer here and `@alice`"+ result = extract(body)++ expect(extractor.engine_refusals).to be_empty+ expect(buffer.quotes).to be_empty+ expect(buffer.mentions.size).to eq(1)+ expect(result).to eq(+ "[quote=\"#{buffer.mentions.first[:placeholder]}\"]\nno closer here and `@alice`",+ )+ end++ it "confirms an angle-bracket autolink in a body with CR line endings" do+ raw = "Subject: Re\r\n\r\n <https://#{source_host}> \r\n\r\nthanks\r\n"+ result = extract(raw)++ expect(extractor.engine_refusals).to be_empty+ expect(buffer.links.first).to include(target_type: enums::LinkTarget::SITE)+ expect(result).to eq(raw.sub("https://#{source_host}", buffer.links.first[:placeholder]))+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/folded_text_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/folded_text_spec.rbnew file mode 100644index 00000000000..fda785d7358--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/folded_text_spec.rb@@ -0,0 +1,87 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::MarkdownScanner::FoldedText do+ def span_for(folded, needle)+ index = folded.text.byteindex(needle)+ raise "#{needle.inspect} not in #{folded.text.inspect}" if index.nil?++ folded.raw_span(index, needle.bytesize)+ end++ describe ".fold" do+ it "compares sigma spellings broadly only for locating raw occurrences" do+ expect(%w[ΚΟΣΜΟΣ κοσμος κοσμοσ].map { |name| described_class.fold(name) }).to eq(+ %w[κοσμοσ κοσμοσ κοσμοσ],+ )+ end++ it "downcases" do+ expect(described_class.fold("@Bob")).to eq("@bob")+ expect(described_class.fold(":MYEMOJI:")).to eq(":myemoji:")+ end++ it "composes, so a decomposed spelling folds to the name it denotes" do+ expect(described_class.fold("#Café")).to eq(described_class.fold("#café"))+ end+ end++ describe "an ASCII body" do+ it "folds to the same byte offsets" do+ folded = described_class.new("Hi @Bob and `@bob`")++ expect(folded.text).to eq("hi @bob and `@bob`")+ expect(span_for(folded, "@bob")).to eq([3, 4])+ expect(folded.raw_span(13, 4)).to eq([13, 4])+ end+ end++ describe "a body with multibyte characters" do+ it "maps a folded span back to the raw bytes it came from" do+ raw = "ping @CAFÉ_team now"+ folded = described_class.new(raw)++ offset, length = span_for(folded, "@café_team")+ expect(raw.byteslice(offset, length)).to eq("@CAFÉ_team")+ end++ it "maps a composed folded span back to its decomposed raw spelling" do+ decomposed = "café"+ raw = "tagged ##{decomposed} here"+ folded = described_class.new(raw)++ offset, length = span_for(folded, "#café")+ expect(raw.byteslice(offset, length)).to eq("##{decomposed}")+ # The raw spelling is one byte longer than the composed name it folds to.+ expect(length).to eq(7)+ end++ it "maps a span past a cluster that folds shorter" do+ # The Kelvin sign downcases to a plain `k`, so every folded byte after it+ # sits at a different offset than in the raw.+ raw = "K @bob"+ folded = described_class.new(raw)++ offset, length = span_for(folded, "@bob")+ expect(raw.byteslice(offset, length)).to eq("@bob")+ end++ it "denotes no raw span for a span that starts or ends inside a cluster" do+ folded = described_class.new("é@bob")++ # The second byte of the folded `é` is not a cluster of its own.+ expect(folded.raw_span(1, 1)).to be_nil+ expect(folded.raw_span(0, 1)).to be_nil+ expect(folded.raw_span(0, 2)).to eq([0, 2])+ end++ it "keeps a grapheme cluster of several codepoints together" do+ family = "👨👩👦"+ folded = described_class.new("#{family}@bob")++ offset, length = folded.raw_span(family.bytesize, 4)+ expect(offset).to eq(family.bytesize)+ expect(length).to eq(4)+ expect(folded.raw_span(1, 4)).to be_nil+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/pathological_input_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/pathological_input_spec.rbnew file mode 100644index 00000000000..061a06344b8--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/pathological_input_spec.rb@@ -0,0 +1,37 @@+# frozen_string_literal: true++# Generated worst-case bodies: repeated malformed construct openers. With unbounded+# construct patterns, each opener re-scans the whole remaining input, and a+# single generated post can occupy a conversion worker for minutes. The+# construct patterns are bounded (line-bounded classes, atomic groups, length+# caps), and these bodies cost a fraction of a second.+#+# This spec only asserts termination within a generous ceiling: a quadratic+# regression exceeds it by orders of magnitude, while the bounded patterns+# plus the engine parse stay around half a second combined. Growth ratios and+# wall-clock scaling are reported by the measurement tooling on the+# `mt/markdown-validation-tooling` branch, where sizes, ratios and GC state+# can be reported instead of asserted against CI noise.+RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ let(:fragments) do+ [+ "[quote=x",+ " } # warmup++ started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)+ fragments.each { |fragment| extract(fragment * 2_000) }+ elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at++ expect(elapsed).to be < 10.0+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/route_parser_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/route_parser_spec.rbnew file mode 100644index 00000000000..32f0977731e--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/route_parser_spec.rb@@ -0,0 +1,56 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::MarkdownScanner::Constructs::InternalLink::RouteParser do+ describe ".parse" do+ {+ "/tag/release/42" => "",+ "/tag/c/42" => "",+ "/tag/intersection/42" => "",+ "/tags/release/42/" => "/",+ "/tag/2015/42/l/latest?ascending=true#top" => "/l/latest?ascending=true#top",+ "/tag/42.json?include=details" => ".json?include=details",+ "/tag/release/42.rss" => ".rss",+ }.each do |path, suffix|+ it "reads the numeric tag id from #{path}" do+ target = described_class.parse(path)++ expect(target).to include(target_type: :tag, target_id: 42, target_name: nil)+ expect(path[target[:route_length]..]).to eq(suffix)+ end+ end++ {+ "/c/parent/123/456" => "",+ "/category/parent/123/456/" => "/",+ "/c/123/456" => "",+ "/c/parent/123/456/l/top/weekly?ascending=true#top" => "/l/top/weekly?ascending=true#top",+ "/c/parent/123/456/none/bug/12" => "/none/bug/12",+ "/c/parent/123/456/all" => "/all",+ "/c/parent/123/456/subcategories" => "/subcategories",+ "/c/parent/123/456.json" => ".json",+ }.each do |path, suffix|+ it "reads the final category id from #{path}" do+ target = described_class.parse(path)++ expect(target).to include(target_type: :category, target_id: 456, target_name: nil)+ expect(path[target[:route_length]..]).to eq(suffix)+ end+ end++ it "keeps reserved multi-tag routes out of single-tag id parsing" do+ expect(described_class.parse("/tags/c/support/feature/12")).to be_nil+ expect(described_class.parse("/tags/intersection/release/42")).to include(+ target_type: :tag_intersection,+ target_tag_path: "release/42",+ )+ end++ it "keeps numeric segments in an id-less category slug path" do+ expect(described_class.parse("/c/parent/123/child/l/latest")).to include(+ target_type: :category,+ target_id: nil,+ target_name: "parent:123:child",+ )+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/tier_gate_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/tier_gate_spec.rbnew file mode 100644index 00000000000..be106b217b9--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/tier_gate_spec.rb@@ -0,0 +1,233 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::MarkdownScanner::TierGate do+ subject(:gate) { described_class.new(constructs:) }++ let(:scanner) { Migrations::Converters::Discourse::MarkdownScanner }++ let(:mention_names) do+ Migrations::CompactStringSet.new(+ %w[alice bob].map { |name| Migrations::NameNormalizer.normalize(name) },+ )+ end+ let(:hashtag_names) do+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("support")])+ end+ let(:emoji_names) do+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("parrot")])+ end++ let(:constructs) do+ [+ scanner::Constructs::Upload.new,+ scanner::Constructs::UploadUrl.new,+ scanner::Constructs::Quote.new,+ scanner::Constructs::InternalLink.new(+ hosts: {+ "forum.example.com" => nil,+ },+ base_prefix: nil,+ on_foreign_host: nil,+ ),+ scanner::Constructs::Mention.new(names: mention_names),+ scanner::Constructs::Hashtag.new(names: hashtag_names),+ scanner::Constructs::Emoji.new(names: emoji_names),+ ]+ end++ it "classifies a body without any candidate character as :none" do+ expect(gate.classify("plain prose, nothing here.")).to eq(:none)+ end++ it "classifies a body whose candidates name nothing as :none" do+ expect(gate.classify("hello @stranger, PR #123 and :smile:")).to eq(:none)+ end++ describe "real candidates" do+ it "preserves sigma identity distinctions when probing known names" do+ names = Migrations::CompactStringSet.new(%w[κοσμος])+ gate = described_class.new(constructs: [scanner::Constructs::Mention.new(names:)])++ expect(gate.classify("@κοσμος")).to eq(:engine)+ expect(gate.classify("@κοσμοσ")).to eq(:none)+ expect(gate.classify("@ΚΟΣΜΟΣ")).to eq(:none)+ end++ it "routes only metadata-bearing quote openers to :engine" do+ # A plain [quote] block holds no user/post/topic fields — nothing any+ # extraction path defers (shown in the engine-scanner spec) — while an+ # opener with metadata does, whatever core's tag casing.+ expect(gate.classify("[quote]\njust quoted text\n[/quote]")).to eq(:none)+ expect(gate.classify("[quote=alice]\nx\n[/quote]")).to eq(:engine)+ expect(gate.classify("[QUOTE=\"alice, post:1\"]\nx\n[/QUOTE]")).to eq(:engine)+ end++ it "routes a known mention to :engine" do+ expect(gate.classify("hello @alice")).to eq(:engine)+ end++ it "routes an escaped mention to :engine, because core still cooks it" do+ # markdown-it strips the escape before the mentions rule runs. A gate+ # that read `\\@` as "not a mention" skipped the body silently, and the+ # engine tier never saw the token it was going to have to place.+ expect(gate.classify("say \\@alice now")).to eq(:engine)+ expect(gate.classify("\\@alice")).to eq(:engine)+ end++ it "routes a name spelled in another case to :engine" do+ expect(gate.classify("ping @ALICE")).to eq(:engine)+ expect(gate.classify("filed under #Support")).to eq(:engine)+ expect(gate.classify(":PARROT:")).to eq(:engine)+ end++ it "over-approximates rather than reading a name's syntax" do+ # `#support::channel` is a chat hashtag core leaves as inert text, and a+ # mid-word `@` names nobody either. Both hold a tracked name, so both+ # cost an engine parse — which is the cheap side of the trade.+ expect(gate.classify("chat in #support::channel today")).to eq(:engine)+ expect(gate.classify("mail [email protected]")).to eq(:engine)+ end++ it "routes a known hashtag to :engine" do+ expect(gate.classify("filed under #support")).to eq(:engine)+ end++ it "treats a quote opener as a candidate without probing names" do+ expect(gate.classify(%{[quote="whoever"]\nhi\n[/quote]})).to eq(:engine)+ end++ it "treats an upload reference as a candidate" do+ expect(gate.classify("")).to eq(:engine)+ end++ it "treats supported full upload URLs as candidates, but not generic uploads paths" do+ sha1 = "0123456789abcdef0123456789abcdef01234567"+ expect(gate.classify("/uploads/default/original/2X/#{sha1}.png")).to eq(:engine)+ expect(gate.classify("/uploads/short-url/aZ9.png")).to eq(:engine)+ # A WordPress-style path has neither supported shape. Without the+ # construct check, every such body would pay an engine parse.+ expect(gate.classify("https://blog.example.net/wp-content/uploads/2009/12/a.jpg")).to eq(+ :none,+ )+ end++ it "treats S3/CDN storage URLs without an uploads segment as candidates" do+ sha1 = "0123456789abcdef0123456789abcdef01234567"+ expect(gate.classify("pic https://cdn.example.com/original/1X/#{sha1}.png")).to eq(:engine)+ expect(+ gate.classify("pic //cdn.example.com/bucket/optimized/2X/a/#{sha1}_2_690x388.png"),+ ).to eq(:engine)+ end++ it "leaves prose that mentions a storage marker without any URL at :none" do+ # The S3 shape needs a host, so `original/` in running text (a path in+ # a code sample, an English sentence) must not pay an engine parse.+ expect(gate.classify("copy the file into original/ before optimizing")).to eq(:none)+ expect(gate.classify("the optimized/ directory holds the variants")).to eq(:none)+ end++ it "treats an internal route as a candidate" do+ expect(gate.classify("see /t/some-topic/123")).to eq(:engine)+ end++ it "treats a configured host as a candidate" do+ expect(gate.classify("https://forum.example.com/faq")).to eq(:engine)+ end++ it "counts a custom emoji as a candidate but not a standard one" do+ expect(gate.classify(":parrot:")).to eq(:engine)+ expect(gate.classify(":smile:")).to eq(:none)+ end++ it "requires the shortcode's closing colon, so a bare name is no candidate" do+ # `:` is far too common to make a candidate of every tracked name that+ # follows one.+ expect(gate.classify("see parrot: the bird")).to eq(:none)+ expect(gate.classify("see :parrot the bird")).to eq(:none)+ end++ it "probes candidates the same next to context-sensitive syntax" do+ expect(gate.classify("`code` and @alice")).to eq(:engine)+ expect(gate.classify("`code` and @stranger and #nothing and :smile:")).to eq(:none)+ end+ end++ describe "character entities" do+ it "routes a numeric entity that decodes to a construct character to :engine" do+ # `@` is `@`: the byte checks cannot see the mention it spells.+ expect(gate.classify("@bob")).to eq(:engine)+ expect(gate.classify("@bob")).to eq(:engine)+ end++ it "ignores a numeric entity that decodes to typography" do+ # `’` is a right single quotation mark; its literal `#` matches+ # the presence check, so only the decode keeps this body out.+ expect(gate.classify("it’s fine")).to eq(:none)+ end++ it "ignores the allowlisted named entities" do+ expect(gate.classify("Tom & Jerry")).to eq(:none)+ expect(gate.classify("wait … more")).to eq(:none)+ end++ it "routes an unlisted named entity to :engine" do+ # `@` is `@`; unknown names are assumed construct-capable.+ expect(gate.classify("@bob")).to eq(:engine)+ expect(gate.classify("½ of it")).to eq(:engine)+ end+ end++ describe "the named-entity allowlist" do+ # The gate's allowlist claims these names can never spell a construct.+ # markdown-it's own name table ships only as an encoded trie, so the claim+ # is verified against the decoder of the engine the scanner runs: every+ # allowlisted name must decode to characters no construct can contain (or+ # not decode at all — a non-entity can't spell anything either).+ it "only lists names markdown-it decodes to non-construct characters" do+ construct_char = described_class.const_get(:CONSTRUCT_CHAR)+ allowlist = described_class::IRRELEVANT_NAMED_ENTITIES++ decoded =+ MarkdownEngineHelper.eval_js(+ "#{allowlist.to_json}.map((name) => __pt.options.engine.utils.unescapeAll(`&${name};`))",+ )++ offenders =+ allowlist+ .zip(decoded)+ .filter_map do |name, text|+ name if text != "&#{name};" && text.each_char.any? { |c| construct_char.match?(c) }+ end++ expect(offenders).to be_empty++ # Positive control: the mechanism must see a construct-capable name.+ expect(+ MarkdownEngineHelper.eval_js('__pt.options.engine.utils.unescapeAll("@")'),+ ).to eq("@")+ end+ end++ describe "an unfamiliar construct" do+ let(:constructs) do+ stub =+ Class.new(Migrations::Converters::Discourse::MarkdownScanner::Constructs::Base) do+ const_set(:TRIGGERS, ["%"].freeze)++ def detect(_input, _pos, _byte)+ nil+ end++ def presence_pattern+ /%pct/+ end+ end+ [stub.new]+ end++ it "assumes candidacy whenever presence hits, so classification stays conservative" do+ expect(gate.classify("%pct")).to eq(:engine)+ expect(gate.classify("plain")).to eq(:none)+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/mention_classifier_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/mention_classifier_spec.rbnew file mode 100644index 00000000000..2ce57172c69--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/mention_classifier_spec.rb@@ -0,0 +1,61 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::MentionClassifier do+ let(:enums) { Migrations::Database::IntermediateDB::Enums }++ it "classifies a plain name as a user mention" do+ expect(described_class.new.call("gerhard")).to eq(enums::MentionType::USER)+ end++ it "classifies @all as an all mention" do+ expect(described_class.new.call("all")).to eq(enums::MentionType::ALL)+ expect(described_class.new.call("All")).to eq(enums::MentionType::ALL)+ end++ describe "here mentions" do+ it "recognizes the default here_mention name" do+ expect(described_class.new.call("here")).to eq(enums::MentionType::HERE)+ end++ it "honors a custom here_mention setting value" do+ classifier = described_class.new(here_mention: "staff")++ expect(classifier.call("staff")).to eq(enums::MentionType::HERE)+ expect(classifier.call("here")).to eq(enums::MentionType::USER)+ end++ it "disables here-detection when here_mention is blank" do+ expect(described_class.new(here_mention: nil).call("here")).to eq(enums::MentionType::USER)+ end+ end++ describe "group mentions" do+ subject(:classifier) { described_class.new(group_names: %w[Admins Moderators]) }++ it "keeps a user distinct from a group with the other lowercase sigma" do+ classifier = described_class.new(group_names: %w[κοσμος])++ expect(classifier.call("κοσμος")).to eq(enums::MentionType::GROUP)+ expect(classifier.call("κοσμοσ")).to eq(enums::MentionType::USER)+ end++ it "recognizes a source group name, case-insensitively" do+ expect(classifier.call("admins")).to eq(enums::MentionType::GROUP)+ expect(classifier.call("Moderators")).to eq(enums::MentionType::GROUP)+ end++ it "treats an unknown name as a user mention" do+ expect(classifier.call("gerhard")).to eq(enums::MentionType::USER)+ end++ # Same name, two Unicode forms: the group is stored decomposed (NFD), the+ # mention typed composed (NFC). Plain downcase would treat them as different;+ # normalization makes them match.+ it "matches regardless of Unicode encoding" do+ name = "Café"+ classifier = described_class.new(group_names: [name.unicode_normalize(:nfd)])++ expect(classifier.call(name.unicode_normalize(:nfc))).to eq(enums::MentionType::GROUP)+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/code_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/code_parity_spec.rbnew file mode 100644index 00000000000..3c40bd43c97--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/code_parity_spec.rb@@ -0,0 +1,260 @@+# frozen_string_literal: true++# Cross-checks the extractor's code handling — the tier gate's routing plus the+# engine tier's count matching — against what core actually treats as code.+# Every row plants a sentinel embed — a mention, or an upload where the+# construct swallows an `@` for other reasons — inside or around one construct,+# and asserts the extractor defers exactly when `PrettyText.cook` renders the+# sentinel. Cooked HTML has no notion of "is byte X code", but a sentinel that+# survives cooking shows that the engine parsed that spot as content, which is the+# oracle extraction must agree with.+#+# CommonMark's own `spec.txt` is deliberately not vendored: it asserts HTML, and+# Discourse's engine diverges from it anyway (`[code]`, `<pre>`, bbcode paragraph+# interruption). Its relevant sections — indented code, fenced code, HTML blocks+# type 1, code spans, list items containing code — are hand-translated into rows+# below and so still covered, through the Discourse oracle.+#+# Needs a booted Rails environment, so it is tagged `:rails` and runs only under+# `MIGRATIONS_RAILS=1`.+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do+ include_context "with parity extractor"++ # The mention sentinel only defers when the source has its name.+ let(:mention_names) do+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("alice")])+ end++ # The sentinels the rows plant. A mention is the cheap one; an upload is+ # used where a construct swallows an `@` for reasons of its own.+ def mention+ "@alice"+ end++ def upload+ ""+ end++ # Each row is [label, raw]. Verified against PrettyText, 2026-08-01.+ def rows+ [+ # --- <code>, which core does NOT treat as code for mentions ---+ ["bare <code> tag", "x <code>#{mention}</code> y"],+ ["bare <code> tag at line start", "<code>#{mention}</code>"],+ # --- [code] block form ---+ ["[code] block", "[code]\n#{mention}\n[/code]"],+ ["[code=lang] block", "[code=ruby]\n#{mention}\n[/code]"],+ ["[code lang=x] block", "[code lang=ruby]\n#{mention}\n[/code]"],+ ["[CODE] uppercase", "[CODE]\n#{mention}\n[/CODE]"],+ ["[code] block after prose", "intro\n\n[code]\n#{mention}\n[/code]"],+ ["[code] block interrupting a paragraph", "intro\n[code]\n#{mention}\n[/code]"],+ ["[code] opener with trailing spaces", "[code] \n#{mention}\n[/code]"],+ ["[code] unclosed to EOF", "[code]\n#{mention}\n"],+ ["[code] closer with trailing text", "[code]\n#{mention}\n[/code] x"],+ ["[code] closer indented four", "[code]\n#{mention}\n [/code]"],+ ["[code] closer indented three", "[code]\n#{mention}\n [/code]"],+ ["nested [code] blocks", "[code]\n[code]\n#{mention}\n[/code]\n[/code]"],+ ["unbalanced [code] openers", "[code]\n[code]\n#{mention}\n[/code]"],+ ["unbalanced [code] openers, all unclosed", "[code]\n[code]\n#{mention}\n"],+ ["[code] inside a list item", "- item\n\n [code]\n #{mention}\n [/code]"],+ ["[code] inside a blockquote", "> [code]\n> #{mention}\n> [/code]"],+ ["[code] closer outside the blockquote", "> [code]\n> #{mention}\n[/code]"],+ ["[code] body outside the blockquote", "> [code]\n#{mention}\n> [/code]"],+ ["[/code] alone", "[/code]\n#{mention}"],+ # --- [code] inline form ---+ ["[code] inline on its own line", "[code]#{mention}[/code]"],+ ["[code] inline mid-line", "x [code]#{mention}[/code] y"],+ ["[code] inline with trailing text", "[code]#{mention}[/code] tail"],+ ["[code] inline spanning a newline", "intro [code]#{mention}\nmore[/code] tail"],+ ["[code] inline across a blank line", "intro [code]#{mention}\n\nmore[/code] tail"],+ ["[code] inline unclosed", "see [code]#{mention} ok"],+ ["[code] inline with a mismatched-case closer", "see [code]#{mention}[/CODE] x"],+ ["[code] inline over nested brackets", "see [code]a[b][/code] #{mention}"],+ ["[code] inline in a list item", "- [code]#{mention}[/code]"],+ ["[code] inline in a heading", "# [code]#{mention}[/code]"],+ ["[code](url) filling a line", "[code](https://example.com/x) #{mention} y[/code]"],+ ["[code](url) mid-line", "z [code](https://example.com/x) #{mention} y[/code]"],+ # --- other bbcode tags are not code ---+ ["[quote] block", "[quote=\"bob\"]\n#{mention}\n[/quote]"],+ ["[details] block", "[details=\"x\"]\n#{mention}\n[/details]"],+ # --- <pre> HTML block ---+ ["<pre> block", "<pre>\n#{mention}\n</pre>"],+ ["<pre> one-line", "<pre>#{mention}</pre>"],+ ["<pre> with attributes", "<pre class=\"x\">\n#{mention}\n</pre>"],+ ["<pre> indented three", " <pre>\n#{mention}\n</pre>"],+ ["<pre> indented four", " <pre>\n#{mention}\n</pre>"],+ ["<pre> mid-line", "x <pre>#{mention}</pre> y"],+ ["<pre> unclosed to EOF", "<pre>\n#{mention}\n"],+ ["<pre> inside a blockquote", "> <pre>\n> #{mention}\n> </pre>"],+ ["text after </pre> on the closing line", "<pre>\nx\n</pre> #{mention}"],+ ["text after a closed <pre>", "<pre>\nx\n</pre>\n\n#{mention}"],+ ["<pre> interrupting a paragraph", "intro\n<pre>\n#{mention}\n</pre>"],+ ["<PRE> uppercase", "<PRE>\n#{mention}\n</PRE>"],+ # --- fenced code ---+ ["fenced block", "```\n#{mention}\n```"],+ ["tilde fence", "~~~\n#{mention}\n~~~"],+ ["tilde does not close a backtick fence", "```\n#{mention}\n~~~\n```"],+ ["fence info with a backtick", "``` a`b\n#{mention}\n```"],+ ["tilde fence info with a backtick", "~~~ a`b\n#{mention}\n~~~"],+ ["fence closer with trailing text", "```\nx\n``` tail\n#{mention}"],+ ["fence indented three", " ```\n#{mention}\n ```"],+ ["fence indented four", "Intro\n\n ```\n#{mention}\n ```"],+ ["longer closing fence", "```\n#{mention}\n`````"],+ ["shorter run does not close", "````\n#{mention}\n```\nmore"],+ ["closer indented three closes", "```\nx\n ```\n#{mention}"],+ ["closer indented six does not close", " ```\n#{mention}\n ```"],+ ["fence unclosed to EOF", "```\n#{mention}"],+ ["fence inside a blockquote", "> ```\n> #{mention}\n> ```"],+ ["fence broken by an unprefixed line", "> ```\n#{mention}\n> ```"],+ ["fence inside a list item", "- item\n\n ```\n #{mention}\n ```"],+ ["fence closing a list", "- item\n\n```\n#{mention}\n```"],+ ["text after a closed fence", "```\nx\n```\n\n#{mention}"],+ # --- indented code ---+ ["indented after a blank line", "Intro\n\n #{mention}"],+ ["indented at the start of input", " #{mention}"],+ ["indented as a lazy continuation", "Intro\n #{mention}"],+ ["indented after a heading without a blank line", "# Title\n #{mention}"],+ ["indented after a closed fence without a blank line", "```\nx\n```\n #{mention}"],+ ["indented after a thematic break", "Intro\n***\n\n #{mention}"],+ ["indented after a setext heading", "Intro\n===\n #{mention}"],+ ["indented after a dash underline", "Intro\n-\n #{mention}"],+ ["indented after a two-dash underline", "Intro\n--\n #{mention}"],+ ["indented after an underline in a blockquote", "> Intro\n> -\n> #{mention}"],+ ["indented after a lone dash", "-\n #{mention}"],+ ["indented after a lone equals", "=\n #{mention}"],+ ["indented after two dashes alone", "--\n #{mention}"],+ ["indented after two equals alone", "==\n #{mention}"],+ ["indented after a lone star", "*\n #{mention}"],+ ["indented after a spaced break", "- - -\n #{mention}"],+ ["indented after spaced equals under a paragraph", "Intro\n= =\n #{mention}"],+ ["indented after spaced dashes under a paragraph", "Intro\n- -\n #{mention}"],+ ["indented after a lazy dash in a blockquote", "> Intro\n-\n #{mention}"],+ ["indented after a lazy equals in a blockquote", "> Intro\n=\n #{mention}"],+ ["indented after lazy dashes in a blockquote", "> Intro\n--\n #{mention}"],+ ["indented after a break leaving a blockquote", "> Intro\n---\n #{mention}"],+ ["indented after a lazy equals in a list", "- Intro\n=\n #{mention}"],+ ["lazy [code] in a blockquote", "> Intro\n[code]\n#{mention}\n[/code]"],+ ["two columns after a lazy dash in a blockquote", "> Intro\n-\n #{mention}"],+ ["indented after a lazy dash in a list", "- Intro\n-\n #{mention}"],+ ["indented after a lazy star in a blockquote", "> Intro\n*\n #{mention}"],+ ["indented after a lazy star under a paragraph", "Intro\n*\n #{mention}"],+ ["indented after a lazy ordered item under a paragraph", "Intro\n2. x\n #{mention}"],+ ["indented after a lazy ordered item in a blockquote", "> Intro\n2. x\n #{mention}"],+ ["indented after a lazy heading in a list", "- Intro\n# h\n #{mention}"],+ ["lazy <pre> in a list", "- Intro\n<pre>\n#{mention}\n</pre>"],+ ["lazy [code] in a list", "- Intro\n[code]\n#{mention}\n[/code]"],+ ["indented after an underline with trailing spaces", "Intro\n- \n #{mention}"],+ ["two columns after a lazy dash in a list", "- Intro\n-\n #{mention}"],+ ["indented after a lazy star in a list", "- Intro\n*\n #{mention}"],+ ["indented in a lazy ordered item's second block", "- Intro\n2. x\n\n #{mention}"],+ ["two columns in a lazy ordered item's second block", "> Intro\n2. x\n\n #{mention}"],+ ["tab-indented after a blank line", "Intro\n\n\t#{mention}"],+ ["space then tab reaching column four", "Intro\n\n \t#{mention}"],+ ["two columns is not enough", "Intro\n\n #{mention}"],+ ["indented block with a blank line inside", "Intro\n\n a\n\n #{mention}"],+ ["unindented line closes an indented block", "Intro\n\n a\ntext\n\n #{mention}"],+ ["four columns under a bullet", "- item\n #{mention}"],+ ["four columns under a bullet after a blank line", "- item\n\n #{mention}"],+ ["six columns under a bullet", "- item\n\n #{mention}"],+ ["six columns under a bullet with a wide marker", "- item\n\n #{mention}"],+ ["five columns under a bullet with a wide marker", "- item\n\n #{mention}"],+ ["four columns under a numbered step", "1. Step\n\n #{mention}"],+ ["seven columns under a numbered step", "1. Step\n\n #{mention}"],+ ["tab after a bullet marker", "-\titem\n\n\t\t#{mention}"],+ ["eight columns under a nested bullet", "- a\n\n - b\n\n #{mention}"],+ ["six columns under a nested bullet", "- a\n\n - b\n\n #{mention}"],+ ["indented inside a blockquote", "> Intro\n>\n> #{mention}"],+ ["indented after a blockquote", "> a\n\n #{mention}"],+ ["indented after a list has ended", "- item\n\ntext\n\n #{mention}"],+ [+ "indented right after a fence closed in a list",+ "- item\n\n ```\n x\n ```\n #{mention}",+ ],+ ["a bullet's own line is not code", "- item\n#{mention}"],+ ["blockquote lazy continuation", "> a\n#{mention}"],+ # --- inline span bounds (a backtick opener that must stay literal) ---+ ["backtick span across a plain line", "`x\nplain text\n#{mention}`"],+ ["backtick span across a blank line", "`x\n\n#{mention}`"],+ ["backtick span across a heading", "`x\n# h\n#{mention}`"],+ ["backtick span across a setext underline", "`x\n===\n#{mention}`"],+ ["backtick span across a dash underline", "`x\n---\n#{mention}`"],+ ["backtick span across a bullet", "`x\n- b\n#{mention}`"],+ ["backtick span across a plus bullet", "`x\n+ b\n#{mention}`"],+ ["backtick span across a numbered item", "`x\n1. b\n#{mention}`"],+ ["backtick span across a paren-numbered item", "`x\n1) b\n#{mention}`"],+ ["backtick span across a later-numbered item", "`x\n2. b\n#{mention}`"],+ ["backtick span across a lone star", "`x\n*\n#{mention}`"],+ ["backtick span across a blockquote", "`x\n> b\n#{mention}`"],+ ["backtick span across a thematic break", "`x\n***\n#{mention}`"],+ ["backtick span across a table", "`x\na|b\n-|-\nc|d\n#{mention}`"],+ ["backtick span across a bbcode opener", "`x\n[quote=\"bob\"]\n#{mention}`"],+ ["backtick span across a fence opener", "`x\n```\n#{mention}`"],+ ["backtick span across a <pre> opener", "`x\n<pre>\n#{mention}`"],+ ["backtick span across an indented line", "`x\n y\n#{mention}`"],+ ["backtick span inside a blockquote", "> `x\n> y\n> #{mention}`"],+ ["backtick span over a blockquote's lazy line", "> `x\ny\n> #{mention}`"],+ # --- multibyte padding, for byte-offset discipline ---+ ["multibyte before an indented block", "Ümläut\n\n #{mention}"],+ ["multibyte inside a fence", "```\nÜ #{mention}\n```"],+ ["multibyte inside a [code] block", "[code]\nÜ #{mention}\n[/code]"],+ ["multibyte before an inline [code] span", "Ü [code]#{mention}[/code] ok"],+ ["multibyte before a backtick span", "Ü `#{mention}` ok"],+ ["multibyte inside a <pre> block", "<pre>\nÜ #{mention}\n</pre>"],+ ["multibyte inside a list item", "- ïtem\n\n #{mention}"],+ # --- CRLF, which markdown-it normalizes before parsing ---+ ["CRLF fenced block", "```\r\n#{mention}\r\n```\r\n"],+ ["CRLF indented block", "Intro\r\n\r\n #{mention}\r\n"],+ ["CRLF [code] block", "[code]\r\n#{mention}\r\n[/code]\r\n"],+ # --- upload sentinels, the embed whose loss is permanent ---+ ["upload in a fence", "```\n#{upload}\n```"],+ ["upload in a [code] block", "[code]\n#{upload}\n[/code]"],+ ["upload in an inline [code] span", "a [code]#{upload}[/code] b"],+ ["upload in a <pre> block", "<pre>\n#{upload}\n</pre>"],+ ["upload in a backtick span", "a `#{upload}` b"],+ ["upload indented after a blank line", "Intro\n\n #{upload}"],+ ["upload indented under a bullet", "- item\n\n #{upload}"],+ ["upload as a lazy continuation", "Intro\n #{upload}"],+ ["upload after a closed [code] block", "[code]\nx\n[/code]\n\n#{upload}"],+ # --- spans over constructs the line-oriented walk once resolved wrong ---+ # An unregistered bbcode tag, an unknown HTML tag, a closing HTML tag,+ # and a table interrupting a paragraph were documented divergences of the+ # deleted line-oriented walk; the engine tier reads them exactly as core.+ ["backtick span across an unregistered bbcode tag", "`x\n[foo]\n#{mention}`"],+ ["backtick span across an unknown HTML tag", "`x\n<pretty>\n#{mention}`"],+ ["backtick span across a closing HTML tag", "`x\n</pre>\n#{mention}`"],+ [+ "indented line below a table interrupting a paragraph",+ "Intro\na|b\n-|-\nc|d\n #{mention}",+ ],+ ]+ end++ # Whether the extractor left the sentinel alone, i.e. read it as code.+ def construct_treats_as_code?(raw)+ buffer = new_buffer+ build_extractor(buffer).extract(raw)+ raw.include?(upload) ? buffer.uploads.empty? : buffer.mentions.empty?+ end++ # Whether core rendered the sentinel, i.e. read it as content. A mention that+ # names nobody still cooks an inert `<span class="mention">`, which is enough:+ # the question is whether the engine saw a mention token at all.+ def core_treats_as_code?(raw)+ cooked = PrettyText.cook(raw)+ marker = raw.include?(upload) ? "<img" : 'class="mention"'+ !cooked.include?(marker)+ end++ it "reads every construct as code exactly when core does" do+ deviations =+ rows.filter_map do |label, raw|+ detected = construct_treats_as_code?(raw)+ cooked = core_treats_as_code?(raw)+ next if detected == cooked++ "#{label} (#{raw.inspect}): tracker=#{detected} core=#{cooked}"+ end++ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/custom_emoji_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/custom_emoji_spec.rbnew file mode 100644index 00000000000..d12a62290f5--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/custom_emoji_spec.rb@@ -0,0 +1,132 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ describe "custom emoji" do+ let(:custom_emoji_names) { %w[parrot +1] }++ it "defers a shortcode that names a source custom emoji" do+ result = extract("nice :parrot: work")++ expect(buffer.emojis.size).to eq(1)+ emoji = buffer.emojis.first+ expect(emoji[:name]).to eq("parrot")+ expect(result).to eq("nice #{emoji[:placeholder]} work")+ end++ it "leaves a standard emoji shortcode as plain text" do+ raw = "hello :smile: there"++ expect(extract(raw)).to eq(raw)+ expect(buffer.emojis).to be_empty+ end++ it "does not treat a clock time as an emoji" do+ raw = "meet at 10:30:45 sharp"++ expect(extract(raw)).to eq(raw)+ expect(buffer.emojis).to be_empty+ end++ it "does not treat a shortcode glued to a word as an emoji" do+ raw = "path:parrot: here"++ expect(extract(raw)).to eq(raw)+ expect(buffer.emojis).to be_empty+ end++ # Core's emoji boundary (`isValidEmojiPrecedingChar`) is narrower than a plain+ # "not a word character" one: it counts only tab/space (markdown-it `isSpace`),+ # a Unicode punctuation/symbol (`isPunctChar`), and a zero-width space. The+ # cases below are verified against PrettyText in `emoji_parity_spec.rb`; kept+ # here as fixtures so the fast suite pins each one.+ [+ ["no-break space", "a\u00A0:parrot: b"],+ ["ideographic space", "a\u3000:parrot: b"],+ ["superscript two", "²:parrot: b"],+ ["vulgar half", "½:parrot: b"],+ ["soft hyphen", "a\u00AD:parrot: b"],+ ["escaping backslash", "a\\:parrot: b"],+ ].each do |label, raw|+ it "leaves a shortcode after a #{label} literal, like core" do+ expect(extract(raw)).to eq(raw)+ expect(buffer.emojis).to be_empty+ end+ end++ it "defers a shortcode after a zero-width space, like core" do+ extract("a\u200B:parrot: b")++ expect(buffer.emojis.first[:name]).to eq("parrot")+ end++ it "defers a shortcode at the start of a line, like core" do+ result = extract("first line\n:parrot: still here")++ expect(buffer.emojis.first[:name]).to eq("parrot")+ expect(result).to include("first line\n#{buffer.emojis.first[:placeholder]} still here")+ end++ it "defers a shortcode right after an opening paren" do+ extract("(:parrot:)")++ expect(buffer.emojis.first[:name]).to eq("parrot")+ end++ it "defers every shortcode of an adjacent chain" do+ result = extract("well done :parrot::+1:")++ expect(buffer.emojis.map { |emoji| emoji[:name] }).to eq(%w[parrot +1])+ placeholders = buffer.emojis.map { |emoji| emoji[:placeholder] }+ expect(result).to eq("well done #{placeholders.join}")+ end++ it "defers a custom emoji chained onto a standard one" do+ result = extract("thanks :smile::parrot:")++ expect(buffer.emojis.map { |emoji| emoji[:name] }).to eq(%w[parrot])+ expect(result).to eq("thanks :smile:#{buffer.emojis.first[:placeholder]}")+ end++ it "does not extract a custom emoji inside a fenced code block" do+ raw = <<~MD+ real :parrot:++ ```+ code :parrot: here+ ```+ MD++ result = extract(raw)++ expect(buffer.emojis.size).to eq(1)+ expect(result).to include("code :parrot: here")+ end++ it "skips emoji detection entirely when the source has no custom emoji" do+ plain_extractor =+ described_class.new(embeds: buffer, mention_names:, hashtag_names:, markdown_engine:)+ raw = "a :parrot: and :smile:"++ expect(plain_extractor.extract(raw)).to eq(raw)+ expect(buffer.emojis).to be_empty+ end++ it "leaves a toned shortcode literal even when a custom emoji shadows the name" do+ # `:parrot:t4:` cooks as the toned standard emoji when one exists — a tone+ # suffix never resolves to a custom emoji — so the text must stay as written.+ raw = "wave :parrot:t4: here"++ expect(extract(raw)).to eq(raw)+ expect(buffer.emojis).to be_empty+ end++ it "defers the custom emoji when the tone-like text has no closing colon" do+ result = extract("wave :parrot:t4 here")++ expect(buffer.emojis.first[:name]).to eq("parrot")+ expect(result).to eq("wave #{buffer.emojis.first[:placeholder]}t4 here")+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/emoji_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/emoji_parity_spec.rbnew file mode 100644index 00000000000..503ddfaefd8--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/emoji_parity_spec.rb@@ -0,0 +1,126 @@+# frozen_string_literal: true++# Cross-checks the custom-emoji construct against what core actually renders. For+# every boundary character in a representative set we build `a<char>:parrot: x`+# (and the forward variant `a :parrot:<char> x`) and assert the construct defers+# exactly when `PrettyText.cook` renders the source's custom emoji.+#+# The deliberate divergence: this construct defers only the source's *custom*+# emoji. A standard shortcode (`:smile:`) cooks in core too, but it needs no+# remapping and round-trips verbatim, so we leave it literal on purpose. The+# core-side predicate is therefore "core rendered THE CUSTOM emoji" — an+# `emoji-custom` image for the test name — with a real CustomEmoji in the DB, not+# just "core rendered some emoji".+#+# Core's emoji rule uses its own boundary (`isValidEmojiPrecedingChar`: markdown-+# it's narrow `isSpace`, `isPunctChar`, or a zero-width space), not the wider+# whitespace-or-punctuation boundary mentions and hashtags run through, so the two+# sides are checked here against PrettyText rather than read off the rule's regex.+# Needs a booted Rails environment, so it is tagged `:rails` and runs only under+# `MIGRATIONS_RAILS=1`.+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do+ include_context "with parity extractor"++ before do+ SiteSetting.enable_emoji = true+ # The mode where the preceding-character boundary is enforced (with inline+ # emoji translation on, core skips it and any character opens a shortcode).+ SiteSetting.enable_inline_emoji_translation = false+ end++ # A real CustomEmoji is what core cooks into an `emoji-custom` image; a standard+ # or unknown shortcode does not. Clearing the cache makes the fresh record+ # visible to PrettyText.+ def create_custom_emoji(name)+ CustomEmoji.create!(name:, upload: Fabricate(:upload))+ Emoji.clear_cache+ end++ before { create_custom_emoji("parrot") }++ def markdown_engine_for(name)+ MarkdownEngineHelper.context_for_names(hashtag_names: [], custom_emoji_names: [name])+ end++ def construct_extracts?(raw, name = "parrot")+ buffer = new_buffer+ build_extractor(+ buffer,+ markdown_engine: markdown_engine_for(name),+ custom_emoji_names: [name],+ ).extract(raw)+ buffer.emojis.any? { |emoji| emoji[:name] == name }+ end++ # Core cooks a custom emoji into an `<img class="emoji emoji-custom" …>` whose+ # title is `:name:`. A standard shortcode cooks a plain `emoji` image (no+ # `emoji-custom`), so this tells apart the source's own emoji.+ def core_cooks?(raw, name = "parrot")+ html = PrettyText.cook(raw)+ html.include?('class="emoji emoji-custom"') && html.include?(%(title=":#{name}:"))+ end++ def deviations_for(direction)+ BoundaryCorpus+ .chars(zero_width_space: true)+ .filter_map do |label, char|+ raw = direction == :before ? "a#{char}:parrot: x" : "a :parrot:#{char} x"+ extracted = construct_extracts?(raw)+ cooked = core_cooks?(raw)+ next if extracted == cooked++ "#{direction} #{label} #{BoundaryCorpus.describe(char)}: construct=#{extracted} core=#{cooked}"+ end+ end++ it "defers exactly when core cooks, for every character before the opening `:`" do+ deviations = deviations_for(:before)+ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end++ it "defers exactly when core cooks, for every character after the closing `:`" do+ deviations = deviations_for(:forward)+ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end++ it "defers a shortcode at the very start of the input, matching core" do+ raw = ":parrot: x"+ expect(construct_extracts?(raw)).to eq(core_cooks?(raw))+ end++ it "defers each shortcode of an adjacent chain, matching core" do+ raw = "look :parrot::parrot: x"+ expect(construct_extracts?(raw)).to be(true)+ expect(core_cooks?(raw)).to be(true)+ end++ it "keeps a toned shortcode literal, matching core" do+ # `:parrot:t4:` resolves to a toned standard emoji, never the custom one, so+ # both sides leave it as written.+ raw = "wave :parrot:t4: here"+ expect(construct_extracts?(raw)).to be(false)+ expect(core_cooks?(raw)).to be(false)+ end++ it "defers a tone-like suffix that has no closing colon, matching core" do+ raw = "wave :parrot:t4 here"+ expect(construct_extracts?(raw)).to be(true)+ expect(core_cooks?(raw)).to be(true)+ end++ it "leaves a `:name:` inside a clock time literal, matching core" do+ # A digit before the `:` is not a boundary, so `10:30:45` never opens a+ # shortcode — even with a custom emoji named after the middle segment.+ create_custom_emoji("30")+ raw = "meet at 10:30:45 sharp"+ expect(construct_extracts?(raw, "30")).to be(false)+ expect(core_cooks?(raw, "30")).to be(false)+ end++ it "defers a custom emoji whose name holds `+` and digits, matching core" do+ create_custom_emoji("+1")+ raw = "nice :+1: work"+ expect(construct_extracts?(raw, "+1")).to be(true)+ expect(core_cooks?(raw, "+1")).to be(true)+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/extraction_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/extraction_spec.rbnew file mode 100644index 00000000000..4d2be8cf771--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/extraction_spec.rb@@ -0,0 +1,583 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ it "returns nil for a nil body" do+ expect(extract(nil)).to be_nil+ end++ it "leaves a body with no embeds untouched" do+ raw = "Just some **plain** text with a (paren) and a / slash."++ expect(extract(raw)).to eq(raw)+ expect(buffer).to be_empty+ end++ # The whole reason to wrap Markbridge's scanner: things that only look like+ # embeds inside code must be left alone.+ describe "code blocks" do+ it "does not extract from a fenced code block" do+ raw = <<~MD+ real @alice here++ ```+ not a @mention and  and [quote="ghost"]q[/quote]+ ```+ MD++ result = extract(raw)++ expect(buffer.mentions.map { |m| m[:name] }).to eq(%w[alice])+ expect(buffer.uploads).to be_empty+ expect(buffer.quotes).to be_empty+ expect(result).to include("not a @mention and ")+ end++ # Inline code spans are covered from the count-matching angle in+ # engine_scanner_spec, placeholder positions included.+ end++ # Indented code opens wherever no paragraph is open and the line reaches four+ # columns past its container's content column — under a bullet that is six+ # columns, under `1.` seven. An indented line inside an open paragraph is a lazy+ # continuation instead. Checked against PrettyText.+ describe "indented code blocks" do+ let(:upload) { "" }++ it "does not extract from an indented block after a blank line" do+ raw = "Intro\n\n #{upload}\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "does not extract from an indented block at the start of the input" do+ raw = " #{upload}\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ # Nothing needs a blank line before it: a heading is a leaf block that closes+ # itself, so the next line opens code straight away.+ it "does not extract from an indented block right under a heading" do+ raw = "# Title\n #{upload}\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "does not extract from an indented block right after a closed fence" do+ raw = "```\nx\n```\n #{upload}\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "does not extract from a tab-indented block" do+ raw = "Intro\n\n\t#{upload}\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "reopens a block after an unindented line interrupts it" do+ raw = "Intro\n\n a\ntext\n\n #{upload}\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ # An indented line straight after a paragraph line is a lazy continuation, so+ # core renders the image rather than a code block.+ it "extracts from an indent that cannot interrupt a paragraph" do+ extract("Intro line\n #{upload}\n")++ expect(buffer.uploads.size).to eq(1)+ end++ it "extracts from an indent under a bullet" do+ extract("- item\n #{upload}\n")++ expect(buffer.uploads.size).to eq(1)+ end++ # Four spaces against a numbered item's content indent of three is one past+ # it, not four, so this renders as the list item's image in core. An indented+ # screenshot under a numbered step is ordinary post formatting.+ it "extracts from an indent under a numbered step" do+ extract("1. Step one\n\n #{upload}\n\n2. Step two\n")++ expect(buffer.uploads.size).to eq(1)+ end++ # Six columns is the bullet's content indent of two plus four, so core does+ # render this one as code.+ it "does not extract from a deeper indent inside a list" do+ raw = "- item\n\n #{upload}\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "does not extract from a tab-indented block inside a list" do+ raw = "-\titem\n\n\t\t#{upload}\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "does not extract from an indented block inside a blockquote" do+ raw = "> Intro\n>\n> #{upload}\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "recognizes a block again once the list has ended" do+ raw = "- item\n\ntext\n\n #{upload}\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end+ end++ # `[code]` is a bbcode tag core renders as code in two shapes: a block, when the+ # opener stands alone on its line and a `[/code]` line closes it, and an inline+ # span otherwise. Every expectation here was checked against PrettyText.+ describe "[code] tags" do+ let(:upload) { "" }++ it "does not extract from a [code] block" do+ raw = "[code]\n#{upload}\n[/code]\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "does not extract from a [code=lang] block" do+ raw = "[code=ruby]\n#{upload}\n[/code]\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "does not extract from an inline [code] span" do+ raw = "see [code]#{upload}[/code] there"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ # The block form declines a closer that does not fill its line, and the inline+ # form — which is bounded by the paragraph, not the line — takes over.+ it "does not extract from a span whose closer has text after it" do+ raw = "[code]\n#{upload}\n[/code] x\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "does not extract from a [code] block inside a blockquote" do+ raw = "> [code]\n> #{upload}\n> [/code]\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "extracts from an unclosed [code], which core leaves as prose" do+ extract("[code]\n#{upload}\n")++ expect(buffer.uploads.size).to eq(1)+ end++ it "extracts after a [code] block has closed" do+ extract("[code]\nx\n[/code]\n\n#{upload}\n")++ expect(buffer.uploads.size).to eq(1)+ end++ # A `[` that a construct claims never reaches the bbcode check, mirroring+ # markdown-it running the link rule before the inline bbcode rule.+ it "extracts from a link whose label happens to be code" do+ extract("z [code](https://external.com/x) #{upload} y[/code]")++ expect(buffer.uploads.size).to eq(1)+ end+ end++ # A `<pre>` at the start of a line is an HTML block, and everything through the+ # line holding `</pre>` is raw HTML. Checked against PrettyText.+ describe "<pre> blocks" do+ let(:upload) { "" }++ it "does not extract from a <pre> block" do+ raw = "<pre>\n#{upload}\n</pre>\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "does not extract from a one-line <pre>" do+ raw = "<pre>#{upload}</pre>\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ # Core's HTML block runs to the end of the input when it is never closed, so+ # the rest of the post is code too.+ it "does not extract from an unclosed <pre>" do+ raw = "<pre>\n#{upload}\n"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "extracts after a <pre> block has closed" do+ extract("<pre>\nx\n</pre>\n\n#{upload}\n")++ expect(buffer.uploads.size).to eq(1)+ end++ # Mid-line it is inline HTML, not a block, so core cooks what is inside it.+ it "extracts from a <pre> in the middle of a line" do+ extract("x <pre>#{upload}</pre> y")++ expect(buffer.uploads.size).to eq(1)+ end+ end++ # Core runs mentions, hashtags and emoji through the text-post-process engine,+ # which skips any text token inside a link — and a link's destination is not a+ # text token at all. So nothing is detected inside a link, and a `@name` there+ # stays somebody else's URL instead of being rewritten to a destination+ # username. Every expectation here was checked against PrettyText.+ describe "links" do+ let(:hashtag_names) do+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("general")])+ end+ let(:internal_link_hosts) { { source_host => nil } }++ [+ "[link](https://external.com/@bob)",+ "see https://external.com/x/@bob now",+ "<https://external.com/@bob>",+ "https://external.com/?a=1&t=@bob",+ "[hi @bob](https://external.com/)",+ "",+ "//external.com/@bob",+ # `_` is a linkify boundary, so core links this one too.+ "x_https://external.com/@bob",+ ].each do |raw|+ it "leaves a mention inside #{raw} alone" do+ expect(extract(raw)).to eq(raw)+ expect(buffer.mentions).to be_empty+ end+ end++ it "leaves a hashtag inside a link alone" do+ raw = "see https://external.com/x/#general now"++ expect(extract(raw)).to eq(raw)+ expect(buffer.hashtags).to be_empty+ end++ # A destination may be followed by a title, and padded, and wrapped in angle+ # brackets. Core cooks all of these as links, so nothing inside them counts.+ [+ %([x](https://external.com/@bob "title")),+ "[x](https://external.com/@bob 'title')",+ "[x](https://external.com/@bob (title))",+ %([x](<https://external.com/@bob> "title")),+ "[x](https://external.com/@bob )",+ "[x]( https://external.com/@bob )",+ %([x](https://external.com/@bob\n"title")),+ %(),+ # The title itself is an attribute, never text.+ %([x](/some/path "@bob title")),+ # Malformed enough that core builds no markdown link — but then linkify+ # takes the destination, so the mention is still inside a link.+ %([x](https://external.com/@bob "unclosed)),+ %([x](https://external.com/@bob extra "t")),+ ].each do |raw|+ it "leaves a mention alone in #{raw.inspect}" do+ expect(extract(raw)).to eq(raw)+ expect(buffer.mentions).to be_empty+ end+ end++ # Core builds no link here and does not linkify anything, so the mention is+ # ordinary text and must still be deferred.+ it "still defers a mention in parentheses that are not a link" do+ extract("[x](not a link @bob)")++ expect(buffer.mentions.map { |mention| mention[:name] }).to eq(%w[bob])+ end++ it "still defers a mention in prose next to a link" do+ extract("hi @bob see https://external.com/@carol now")++ expect(buffer.mentions.map { |mention| mention[:name] }).to eq(%w[bob])+ end++ # The skip must not swallow a link another construct wants, or the embed it+ # carries would go unrecorded.+ it "still defers an internal link" do+ extract("[t](https://forum.example.com/t/slug/5)")++ expect(buffer.links.size).to eq(1)+ end++ it "still defers an upload inside a link" do+ extract("[f](https://cdn.example.com/uploads/default/original/1X/#{sha1}.png)")++ expect(buffer.uploads.size).to eq(1)+ end++ # The outer bracket must not match, or the walk would never reach the inner+ # image and the upload would be lost.+ it "still defers the inner image of a linked image on a foreign host" do+ extract("[](https://external.com/@bob)")++ expect(buffer.uploads.size).to eq(1)+ expect(buffer.mentions).to be_empty+ end++ # A `@name` in a relative destination is inside a link just like an+ # absolute one; a placeholder spliced into the URL would corrupt it.+ it "leaves a mention alone in a nested-text link's relative destination" do+ raw = "[see [1]](/x/@bob) end"++ expect(extract(raw)).to eq(raw)+ expect(buffer.mentions).to be_empty+ end++ # Text nested too deep for any link pattern still leaves its `](…)`+ # destination consumed, not walked into.+ it "leaves a mention alone in the destination of a link too nested to match" do+ raw = "[a [b [c]]](/x/@bob) end"++ expect(extract(raw)).to eq(raw)+ expect(buffer.mentions).to be_empty+ end+ end++ # An unpaired backtick is literal text in CommonMark, so it must not open a code+ # span that swallows the rest of the post. A span exists only when a matching+ # closer follows within the same paragraph. Every expectation here was checked+ # against PrettyText.+ describe "inline code spans" do+ let(:hashtag_names) do+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("general")])+ end+ let(:hashtag_extractor) do+ described_class.new(embeds: buffer, mention_names:, hashtag_names:, markdown_engine:)+ end+ let(:emoji_extractor) do+ described_class.new(+ embeds: buffer,+ markdown_engine:,+ mention_names:,+ hashtag_names:,+ custom_emoji_names: %w[parrot],+ )+ end++ it "extracts a mention after an unpaired backtick" do+ extract("a`@alice here")++ expect(buffer.mentions.map { |m| m[:name] }).to eq(%w[alice])+ end++ it "extracts a hashtag after an unpaired backtick" do+ hashtag_extractor.extract("a`#general here")++ expect(buffer.hashtags.map { |h| h[:name] }).to eq(%w[general])+ end++ it "extracts a custom emoji after an unpaired backtick" do+ emoji_extractor.extract("a`:parrot: here")++ expect(buffer.emojis.map { |e| e[:name] }).to eq(%w[parrot])+ end++ it "suppresses a mention inside a paired span" do+ result = extract("`@alice` here")++ expect(buffer.mentions).to be_empty+ expect(result).to eq("`@alice` here")+ end++ it "suppresses a mention in a span that crosses a single newline" do+ result = extract("`a\n@alice` here")++ expect(buffer.mentions).to be_empty+ expect(result).to eq("`a\n@alice` here")+ end++ it "treats both halves as literal when a blank line splits the backticks" do+ result = extract("`@alice\n\n@bob` here")++ expect(buffer.mentions.map { |m| m[:name] }).to eq(%w[alice bob])+ expect(result).to include("`", "`")+ end++ it "keeps an embedded single backtick inside a double-backtick span as code" do+ result = extract("``a`@alice`` here")++ expect(buffer.mentions).to be_empty+ expect(result).to eq("``a`@alice`` here")+ end++ it "keeps detecting after a double run that finds no matching closer" do+ # ``@alice` — the `` opens no span (no `` closer follows), so it is literal+ # and the mention after it is extracted.+ extract("``@alice`")++ expect(buffer.mentions.map { |m| m[:name] }).to eq(%w[alice])+ end+ end++ it "raises on a node type it has no defer handler for" do+ # `extract` builds its constructs internally and never produces an unknown+ # node, so this guard is unreachable through the public API; open up the+ # private method deliberately to exercise it.+ seam =+ Class+ .new(described_class) { public :defer }+ .new(embeds: buffer, mention_names:, hashtag_names:, markdown_engine:)++ expect { seam.defer(Object.new, "@x") }.to raise_error(NotImplementedError, /Object/)+ end++ # The contract: every token spliced into the result maps to exactly one recorded+ # linkage descriptor.+ it "keeps placeholders and linkage rows one-to-one" do+ result =+ extract(+ "intro @carol see  and\n" \+ "[quote=\"dan, post:9, topic:3\"]\nq\n[/quote] done",+ )++ expect(Migrations::Placeholder.scan(result)).to match_array(buffer.placeholders)+ end++ describe "verbatim originals" do+ # Every deferred row carries the exact matched source slice, so the+ # importer can restore it unchanged when the embed cannot be mapped —+ # rebuilding a canonical form loses titles, spacing and casing.+ it "records the matched slice on every kind of row" do+ extract(<<~RAW, topic_id: 7)+ [quote="alice, post:2"]+ hi @bob, see #support+ [/quote]+ [text](/t/topic/5) and + RAW++ expect(buffer.quotes.first[:original_markdown]).to eq(%q{[quote="alice, post:2"]})+ expect(buffer.mentions.first[:original_markdown]).to eq("@bob")+ expect(buffer.hashtags.first[:original_markdown]).to eq("#support")+ expect(buffer.links.first[:original_markdown]).to eq("[text](/t/topic/5)")+ expect(buffer.uploads.first[:original_markdown]).to eq("")+ end+ end++ describe "Unicode raw" do+ it "leaves a body of only Unicode text untouched" do+ raw = "これは 🎉 café テスト — nothing to extract"++ expect(extract(raw)).to eq(raw)+ expect(buffer).to be_empty+ end++ it "captures a username with a combining mark (decomposed form)" do+ name = "José".unicode_normalize(:nfd)+ extract("ping @#{name} thanks")++ captured = buffer.mentions.first[:name]+ expect(captured.unicode_normalize).to eq("José".unicode_normalize)+ end++ it "captures a CJK username" do+ extract("hi @田中 there")++ expect(buffer.mentions.first[:name]).to eq("田中")+ end++ it "preserves Unicode around an extracted embed and stays valid encoding" do+ result = extract("日本語  🎉")++ expect(buffer.uploads.size).to eq(1)+ expect(result).to eq("日本語 #{buffer.uploads.first[:placeholder]} 🎉")+ expect(result).to be_valid_encoding+ end++ it "does not extract embeds from a code block that contains Unicode" do+ raw = "```\n@josé [quote=\"x, post:1\"] 日本\n```\n@real"+ result = extract(raw)++ expect(buffer.mentions.map { |mention| mention[:name] }).to eq(%w[real])+ expect(result).to include("@josé", '[quote="x, post:1"]', "日本")+ end++ # Multibyte text BEFORE a construct shifts every later byte offset away from+ # its character offset, so any byte-indexed look-back reads the wrong byte.+ # These bodies are shaped so that wrong byte is an alphanumeric — a boundary+ # check that mixes up the two index kinds rejects the construct.+ context "with multibyte text before the construct" do+ it "still defers a mention" do+ result = extract("héllo @alice hi")++ expect(buffer.mentions.first[:name]).to eq("alice")+ expect(result).to eq("héllo #{buffer.mentions.first[:placeholder]} hi")+ end++ it "still defers a hashtag" do+ extract("höhe #support da")++ expect(buffer.hashtags.first[:name]).to eq("support")+ end++ it "still defers a bare internal link" do+ # An absolute bare URL still detects in prose, so it exercises the+ # byte-offset look-back with multibyte text before it.+ host_extractor =+ described_class.new(+ embeds: buffer,+ markdown_engine:,+ mention_names:,+ hashtag_names:,+ internal_link_hosts: {+ "forum.example.com" => nil,+ },+ )+ host_extractor.extract("Höhe https://forum.example.com/t/thema/9 an")++ expect(buffer.links.first).to include(target_id: 9)+ end++ it "still defers a custom emoji" do+ emoji_extractor =+ described_class.new(+ embeds: buffer,+ markdown_engine:,+ mention_names:,+ hashtag_names:,+ custom_emoji_names: %w[parrot],+ )+ emoji_extractor.extract("schön :parrot:")++ expect(buffer.emojis.first[:name]).to eq("parrot")+ end++ it "still keeps a glued mention literal" do+ raw = "das naïve@alice bleibt"++ expect(extract(raw)).to eq(raw)+ expect(buffer.mentions).to be_empty+ end+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/hashtags_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/hashtags_parity_spec.rbnew file mode 100644index 00000000000..bf230885b0c--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/hashtags_parity_spec.rb@@ -0,0 +1,63 @@+# frozen_string_literal: true++# Cross-checks the hashtag construct against what core actually renders. For every+# boundary character in a representative set we build `a<char>#general x` (and the+# forward variant `a #general<char> x`) and assert the construct extracts exactly+# when `PrettyText.cook` produces a cooked hashtag link. This needs a booted Rails+# environment (PrettyText's server-side markdown-it and a real category to look+# up), so it is tagged `:rails` and runs only under `MIGRATIONS_RAILS=1`.+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do+ include_context "with parity extractor"++ # The category the battery looks up. A hashtag whose name resolves to a real+ # category is what core cooks into a `hashtag-cooked` link.+ let!(:category) do+ Category.find_by(slug: "general") ||+ Fabricate(:category, name: "General parity", slug: "general")+ end+ let!(:user) { Fabricate(:user) }++ # Extraction is gated on the source's names, so the construct defers only a+ # hashtag that names something real — the same condition under which core cooks.+ let(:hashtag_names) do+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("general")])+ end++ let(:markdown_engine) { MarkdownEngineHelper.context_for_names(hashtag_names: %w[general]) }++ def construct_extracts?(raw)+ buffer = new_buffer+ build_extractor(buffer).extract(raw)+ buffer.hashtags.any?+ end++ def core_cooks?(raw)+ PrettyText.cook(raw, user_id: user.id).include?("hashtag-cooked")+ end++ def deviations_for(direction)+ BoundaryCorpus.chars.filter_map do |label, char|+ raw = direction == :before ? "a#{char}#general x" : "a #general#{char} x"+ extracted = construct_extracts?(raw)+ cooked = core_cooks?(raw)+ next if extracted == cooked++ "#{direction} #{label} #{BoundaryCorpus.describe(char)}: construct=#{extracted} core=#{cooked}"+ end+ end++ it "extracts exactly when core cooks, for every character before the hash" do+ deviations = deviations_for(:before)+ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end++ it "extracts exactly when core cooks, for every character after the name" do+ deviations = deviations_for(:forward)+ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end++ it "extracts a hashtag at the very start of the input, matching core" do+ raw = "#general x"+ expect(construct_extracts?(raw)).to eq(core_cooks?(raw))+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/hashtags_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/hashtags_spec.rbnew file mode 100644index 00000000000..9c45236d429--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/hashtags_spec.rb@@ -0,0 +1,186 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ describe "hashtags" do+ it "defers a bare hashtag, recording the name and leaving the type for import" do+ result = extract("see #announcements please")++ expect(buffer.hashtags.size).to eq(1)+ hashtag = buffer.hashtags.first+ expect(hashtag).to include(name: "announcements", hashtag_type: nil, target_id: nil)+ expect(result).to eq("see #{hashtag[:placeholder]} please")+ end++ it "keeps a category's parent:child separator in the name" do+ extract("in #support:billing here")++ expect(buffer.hashtags.first).to include(name: "support:billing", hashtag_type: nil)+ end++ it "records a forced ::tag suffix as the tag type, dropping the suffix from the name" do+ extract("tagged #release::tag today")++ expect(buffer.hashtags.first).to include(+ name: "release",+ hashtag_type: enums::HashtagType::TAG,+ )+ end++ it "records a forced ::category suffix case-insensitively" do+ extract("filed #Support::CATEGORY now")++ expect(buffer.hashtags.first).to include(+ name: "Support",+ hashtag_type: enums::HashtagType::CATEGORY,+ )+ end++ it "defers a hashtag right after an opening paren" do+ result = extract("(#news)")++ expect(buffer.hashtags.first[:name]).to eq("news")+ expect(result).to eq("(#{buffer.hashtags.first[:placeholder]})")+ end++ it "defers a hashtag right after prose punctuation" do+ ["a comma,#news", 'a quote "#news"', "end of sentence.#news"].each do |raw|+ buffer.clear+ extract(raw)+ expect(buffer.hashtags.first).to include(name: "news"), "expected #{raw.inspect} to extract"+ end+ end++ it "treats a symbol before the hash as a boundary, the way core does" do+ # markdown-it's `isPunctChar` accepts Unicode symbols too, so core renders a+ # hashtag after `€` — extraction has to match.+ extract("price€#news")++ expect(buffer.hashtags.first).to include(name: "news")+ end++ it "keeps a trailing sentence dot outside the name" do+ result = extract("see #news.")++ expect(buffer.hashtags.first[:name]).to eq("news")+ expect(result).to eq("see #{buffer.hashtags.first[:placeholder]}.")+ end++ it "does not treat a markdown heading as a hashtag" do+ raw = "# Heading\n\nbody"++ expect(extract(raw)).to eq(raw)+ expect(buffer.hashtags).to be_empty+ end++ it "does not treat a mid-word # as a hashtag" do+ raw = "issue no#42 was closed"++ expect(extract(raw)).to eq(raw)+ expect(buffer.hashtags).to be_empty+ end++ it "leaves an unknown ::channel-style suffix as literal text" do+ raw = "chat in #general::channel today"++ expect(extract(raw)).to eq(raw)+ expect(buffer.hashtags).to be_empty+ end++ it "does not extract a hashtag inside a fenced code block" do+ raw = <<~MD+ real #announcements++ ```+ not a #hashtag here+ ```+ MD++ result = extract(raw)++ expect(buffer.hashtags.map { |h| h[:name] }).to eq(%w[announcements])+ expect(result).to include("not a #hashtag here")+ end+ end++ describe "the hashtag name gate" do+ let(:hashtag_names) { Migrations::CompactStringSet.new(%w[announcements support:billing]) }++ it "requires the names, so no caller can defer every hashtag by accident" do+ expect { described_class.new(embeds: buffer, mention_names:) }.to raise_error(+ ArgumentError,+ /hashtag_names/,+ )+ end++ it "defers a hashtag whose name is in the set" do+ result = extract("see #announcements please")++ expect(buffer.hashtags.first[:name]).to eq("announcements")+ expect(result).to eq("see #{buffer.hashtags.first[:placeholder]} please")+ end++ it "leaves a hashtag that names nothing on the source as literal text" do+ raw = "tracked in PR #123 and channel #general"++ expect(extract(raw)).to eq(raw)+ expect(buffer.hashtags).to be_empty+ end++ it "defers a parent:child category path in the set" do+ extract("filed under #support:billing today")++ expect(buffer.hashtags.first[:name]).to eq("support:billing")+ end++ it "matches the set case- and Unicode-insensitively" do+ extract("see #Announcements please")++ expect(buffer.hashtags.first[:name]).to eq("Announcements")+ end++ it "leaves a forced suffix on an unknown name as literal text" do+ raw = "tagged #unknown::tag today"++ expect(extract(raw)).to eq(raw)+ expect(buffer.hashtags).to be_empty+ end++ it "gates a forced suffix on the name, deferring a known one" do+ extract("in #announcements::category now")++ expect(buffer.hashtags.first).to include(+ name: "announcements",+ hashtag_type: enums::HashtagType::CATEGORY,+ )+ end++ it "matches a dotted name whole instead of stopping at the dot" do+ dotted =+ described_class.new(+ embeds: buffer,+ markdown_engine:,+ mention_names:,+ hashtag_names: Migrations::CompactStringSet.new(%w[v2.0]),+ )+ dotted.extract("ship #v2.0 today")++ expect(buffer.hashtags.first[:name]).to eq("v2.0")+ end++ it "does not truncate a dotted name to a shorter prefix that is in the set" do+ dotted =+ described_class.new(+ embeds: buffer,+ markdown_engine:,+ mention_names:,+ hashtag_names: Migrations::CompactStringSet.new(%w[v2]),+ )+ raw = "ship #v2.0 today"++ expect(dotted.extract(raw)).to eq(raw)+ expect(buffer.hashtags).to be_empty+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_encoded_names_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_encoded_names_spec.rbnew file mode 100644index 00000000000..7e86b36f044--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_encoded_names_spec.rb@@ -0,0 +1,153 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ # With `unicode_usernames` on, a username, group name or tag name holds+ # characters a URL has to percent-encode. Email notifications write the link+ # that way, and the engine's normalized href is encoded even where the author+ # typed the name literally — so the route grammar has to read the encoded+ # spelling and hand the importer the decoded name it stores.+ describe "internal links with percent-encoded names" do+ let(:internal_link_hosts) { { source_host => nil } }++ it "decodes a user name while keeping the URL's own spelling" do+ raw = "https://forum.example.com/u/dawid_gawe%C5%82"+ link, result = link_for(raw)++ expect(link).to include(+ url: raw,+ target_type: enums::LinkTarget::USER,+ target_id: nil,+ target_name: "dawid_gaweł",+ target_suffix: nil,+ original_markdown: raw,+ url_offset: 0,+ )+ expect(result).to eq(link[:placeholder])+ end++ it "decodes a name written entirely in encoded bytes" do+ link, result = link_for("[x](/u/%EC%9C%A0%EC%86%8C%ED%98%84)")++ expect(link).to include(+ url: "/u/%EC%9C%A0%EC%86%8C%ED%98%84",+ target_type: enums::LinkTarget::USER,+ target_name: "유소현",+ url_offset: 4,+ )+ expect(result).to eq(link[:placeholder])+ end++ it "decodes an encoded byte in the middle of a `/users/` name" do+ link, = link_for("[x](/users/hasan_%C3%B6zdemir/summary)")++ expect(link).to include(target_name: "hasan_özdemir", target_suffix: "/summary")+ end++ it "reads lowercase hex the same way" do+ link, = link_for("[x](/u/dawid_gawe%c5%82)")++ expect(link).to include(target_name: "dawid_gaweł")+ end++ # The two spellings have to reach the importer as the same name, or the+ # encoded one resolves against nothing.+ it "records the same name for the encoded and the literal spelling" do+ encoded, = link_for("[x](/u/hasan_%C3%B6zdemir)")+ buffer.clear+ literal, = link_for("[x](/u/hasan_özdemir)")++ expect(encoded[:target_name]).to eq(literal[:target_name])+ end++ it "keeps a query string out of a decoded name" do+ link, = link_for("[x](/u/dawid_gawe%C5%82?u=x)")++ expect(link).to include(target_name: "dawid_gaweł", target_suffix: "?u=x")+ end++ it "keeps a fragment out of a decoded name" do+ link, = link_for("[x](/u/dawid_gawe%C5%82#bio)")++ expect(link).to include(target_name: "dawid_gaweł", target_suffix: "#bio")+ end++ it "decodes a group name" do+ link, = link_for("[x](/g/te%C3%A4m/members)")++ expect(link).to include(+ target_type: enums::LinkTarget::GROUP,+ target_name: "teäm",+ target_suffix: "/members",+ )+ end++ it "decodes a tag name" do+ link, = link_for("[x](/tag/%C3%A9tude)")++ expect(link).to include(target_type: enums::LinkTarget::TAG, target_name: "étude")+ end++ it "decodes the tag of a category+tag route" do+ link, = link_for("[x](/tags/c/plugin/22/offi%C3%A7ial)")++ expect(link).to include(+ target_type: enums::LinkTarget::CATEGORY_TAG,+ target_id: 22,+ target_tag_path: "offiçial",+ )+ end++ it "decodes every tag of an intersection route" do+ link, = link_for("[x](/tags/intersection/f%C3%B6od/wine)")++ expect(link).to include(+ target_type: enums::LinkTarget::TAG_INTERSECTION,+ target_tag_path: "föod/wine",+ )+ end++ # Core writes an encoded-method slug into the column percent-encoded, so a+ # slug is compared the way it arrives; decoding it would miss the row.+ it "leaves a category slug path encoded" do+ link, = link_for("[x](/c/caf%C3%A9/billing)")++ expect(link).to include(+ target_type: enums::LinkTarget::CATEGORY,+ target_id: nil,+ target_name: "caf%C3%A9:billing",+ )+ end++ it "leaves a topic slug encoded" do+ link, = link_for("[x](/t/caf%C3%A9-time)")++ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_name: "caf%C3%A9-time")+ end++ # Bytes that decode to no valid UTF-8 name no record, and a `%2F` would+ # invent path structure the author never wrote — both refuse rather than+ # rewrite a coordinate-shaped path onto the destination.+ {+ "invalid UTF-8 bytes" => "/u/%FF%FE",+ "an encoded slash" => "/u/a%2Fb",+ "an encoded newline" => "/g/te%0Aam",+ }.each do |label, path|+ it "refuses a name holding #{label}" do+ raw = "see https://forum.example.com#{path} there"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ expect(extractor.engine_refusals).to eq(invalid_internal_route: 1)+ end+ end++ it "still refuses a name holding a character no route grammar admits" do+ raw = "[x](/u/bob!!!)"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_foreign_host_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_foreign_host_spec.rbnew file mode 100644index 00000000000..b972f35c896--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_foreign_host_spec.rb@@ -0,0 +1,86 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ describe "foreign-host internal-link signal" do+ let(:internal_link_hosts) { { source_host => nil } }+ let(:on_foreign_host) { ->(host) { foreign_hosts << host } }+ let(:foreign_hosts) { [] }++ it "fires the callback for an absolute route-shaped link on an unconfigured host" do+ extract("elsewhere https://old-forum.example.com/t/slug/99 done")++ expect(foreign_hosts).to eq(["old-forum.example.com"])+ expect(buffer.links).to be_empty+ end++ it "fires for a foreign-host markdown link too" do+ extract("[a topic](https://old-forum.example.com/t/slug/99)")++ expect(foreign_hosts).to eq(["old-forum.example.com"])+ end++ it "reports a non-default port as part of the host" do+ extract("https://old-forum.example.com:8080/t/slug/99")++ expect(foreign_hosts).to eq(["old-forum.example.com:8080"])+ end++ it "reports each host once, however often it appears" do+ extract("https://old-forum.example.com/t/slug/99")+ extract("https://old-forum.example.com/t/other/100")++ expect(foreign_hosts).to eq(["old-forum.example.com"])+ end++ it "does not fire for a foreign host whose path is not an internal route" do+ extract("see https://old-forum.example.com/about/team here")++ expect(foreign_hosts).to be_empty+ end++ it "does not fire for a configured host" do+ extract("read https://forum.example.com/t/slug/99 now")++ expect(foreign_hosts).to be_empty+ end++ it "does not fire for a relative link" do+ extract("go to /t/slug/99")++ expect(foreign_hosts).to be_empty+ end++ it "treats every absolute route-shaped link as foreign when no host is configured" do+ no_host =+ described_class.new(+ embeds: buffer,+ markdown_engine:,+ mention_names:,+ hashtag_names:,+ on_foreign_host: ->(host) { foreign_hosts << host },+ )+ no_host.extract("read https://any.example.com/t/slug/99 now")++ expect(foreign_hosts).to eq(["any.example.com"])+ end++ it "is a no-op when no callback is given" do+ plain =+ described_class.new(+ embeds: buffer,+ markdown_engine:,+ mention_names:,+ hashtag_names:,+ internal_link_hosts: {+ "forum.example.com" => nil,+ },+ )++ expect(plain.extract("elsewhere https://old-forum.example.com/t/slug/99 done")).to eq(+ "elsewhere https://old-forum.example.com/t/slug/99 done",+ )+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_parity_spec.rbnew file mode 100644index 00000000000..fcdf2e39b9d--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_parity_spec.rb@@ -0,0 +1,126 @@+# frozen_string_literal: true++# Cross-checks the internal-link construct's bare-URL boundary against what core+# actually renders. For every boundary character in a representative set we build+# `a<char>https://forum.example.com/t/slug/5 b` (and the forward variant with the+# character right after the URL) and assert the construct records a link exactly+# when `PrettyText.cook` linkifies an anchor for that URL.+#+# Core's machinery for a bare absolute URL in prose is markdown-it's linkify, fed+# by two engines whose admissions are unioned: the inline linkify rule (a scheme+# after anything outside `[A-Za-z0-9.+-]`) and the core linkify ruler+# (linkify-it, which also admits `.` and `-`). The net boundary before a scheme is "any+# character except an ASCII letter, digit or `+`" (and `\`, a markdown escape) —+# far wider than the whitespace-or-paren gate the construct used to admit at, so it+# is checked here against PrettyText rather than read off a regex. The URL is+# inline in a sentence, so no onebox block path. Needs a booted Rails environment,+# so it is tagged `:rails` and runs only under `MIGRATIONS_RAILS=1`.+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do+ include_context "with parity extractor"++ before { SiteSetting.enable_markdown_linkify = true }++ # The construct treats an absolute URL as internal only when its host is one it+ # was given; core linkifies any absolute URL regardless. Using the source's own+ # host keeps the two comparable — every row's URL is on it.+ def host+ "forum.example.com"+ end++ def url+ "https://#{host}/t/slug/5"+ end++ # What the extractor did with the URL: `:link` (a row was recorded),+ # `:refused` (the body landed on the refusal tally), or `:none`.+ def construct_outcome(raw)+ buffer = new_buffer+ extractor = build_extractor(buffer, internal_link_hosts: { host => nil })+ extractor.extract(raw)+ return :link if buffer.links.any?++ extractor.engine_refusals.values.sum > 0 ? :refused : :none+ end++ # Core linkifies a bare URL into an anchor whose href is the URL. A trailing+ # character right after the URL may extend the href; `extended` reports that,+ # because an extended href is the one case where the extractor may refuse+ # instead of recording a link (the extension can turn the id into a junk+ # coordinate path).+ def core_reading(raw)+ hrefs = PrettyText.cook(raw).scan(/<a\b[^>]*href="([^"]*)"/).flatten+ matching = hrefs.select { |href| href.start_with?(url) }+ { linkified: matching.any?, extended: matching.any? { |href| href.length > url.length } }+ end++ it "records a link exactly when core linkifies, for every character before the URL" do+ deviations =+ BoundaryCorpus.chars.filter_map do |label, char|+ raw = "a#{char}#{url} b"+ outcome = construct_outcome(raw)+ core = core_reading(raw)++ if (outcome != :none) != core[:linkified]+ "before #{label} #{BoundaryCorpus.describe(char)}: " \+ "construct=#{outcome} core=#{core[:linkified]}"+ elsif outcome == :refused+ # Nothing follows the URL here, so its route is intact; a refusal+ # would hide a regression from recording links to refusing them.+ "before #{label} #{BoundaryCorpus.describe(char)}: refused instead of recording"+ end+ end+ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end++ it "records a link exactly when core linkifies, for every character after the URL" do+ deviations =+ BoundaryCorpus.chars.filter_map do |label, char|+ raw = "a #{url}#{char} b"+ outcome = construct_outcome(raw)+ core = core_reading(raw)++ if (outcome != :none) != core[:linkified]+ "forward #{label} #{BoundaryCorpus.describe(char)}: " \+ "construct=#{outcome} core=#{core[:linkified]}"+ elsif outcome == :refused && !core[:extended]+ # A refusal is only right when core swallowed the character into the+ # href and made the trailing id a junk coordinate path. With the+ # href unchanged, refusing instead of recording is a regression.+ "forward #{label} #{BoundaryCorpus.describe(char)}: refused instead of recording"+ end+ end+ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end++ # A trailing ASCII letter or `_` extends the URL's `/t/slug/5` id into `5a` / `5_`,+ # which names no topic — so no route parses. Core linkifies the longer URL+ # all the same, and the extractor sees it whole; but a coordinate-shaped+ # path that parses no route refuses rather than carrying its stale-looking+ # tail onto a new origin, so the body stays verbatim and lands on the+ # refusal tally.+ it "refuses a coordinate-shaped trailing-word URL that core linkifies whole" do+ raw = "a #{url}a b"+ buffer = new_buffer+ extractor = build_extractor(buffer, internal_link_hosts: { host => nil })++ expect(extractor.extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ expect(extractor.engine_refusals).to eq(invalid_internal_route: 1)+ expect(core_reading(raw)[:linkified]).to be(true)+ end++ it "records a link at the very start of the input, matching core" do+ raw = "#{url} b"+ expect(construct_outcome(raw)).to eq(:link)+ expect(core_reading(raw)[:linkified]).to be(true)+ end++ # Core does not linkify a bare relative path (`/t/slug/5`) in prose — linkify+ # only touches schemed and fuzzy-host URLs — so leaving it literal is parity,+ # not a divergence.+ it "leaves a bare relative path literal, matching core" do+ raw = "see /t/slug/5 here"+ expect(construct_outcome(raw)).to eq(:none)+ expect(core_reading(raw)[:linkified]).to be(false)+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_route_tails_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_route_tails_spec.rbnew file mode 100644index 00000000000..744efc3a81b--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_route_tails_spec.rb@@ -0,0 +1,178 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ # What may trail a route without being part of it: the `/` a link generator+ # hangs off the end, and the `.json`/`.rss` extension Rails takes off before+ # it matches. Both stay in the suffix, so the rebuilt URL ends the way the+ # author wrote it.+ describe "internal links with a route tail" do+ let(:internal_link_hosts) { { source_host => nil } }++ it "reads a slug-only topic link that ends in a slash" do+ raw = "https://forum.example.com/t/discourse-tab-bar-for-mobile/"+ link, result = link_for(raw)++ expect(link).to include(+ url: raw,+ target_type: enums::LinkTarget::TOPIC,+ target_id: nil,+ target_name: "discourse-tab-bar-for-mobile",+ target_suffix: "/",+ original_markdown: raw,+ url_offset: 0,+ )+ expect(result).to eq(link[:placeholder])+ end++ it "keeps the slash in front of a slug-only topic link's query" do+ link, = link_for("[x](/t/how-to-fix-it/?page=2)")++ expect(link).to include(target_name: "how-to-fix-it", target_suffix: "/?page=2")+ end++ {+ "a topic with a slug and id" => ["/t/slug/18978/", { target_id: 18_978 }],+ "a category by id" => ["/c/support/6/", { target_id: 6 }],+ "a category by slug path" => ["/c/support/billing/", { target_name: "support:billing" }],+ "a badge" => ["/badges/9/great/", { target_id: 9 }],+ "a category+tag route" => ["/tags/c/food/wine/", { target_tag_path: "wine" }],+ "a tag intersection" => ["/tags/intersection/food/wine/", { target_tag_path: "food/wine" }],+ }.each do |label, (path, fields)|+ it "keeps a trailing slash out of the route for #{label}" do+ link, = link_for("[x](#{path})")++ expect(link).to include(**fields, target_suffix: "/")+ end+ end++ # A digit run that overflows an id is still the numeric-title ambiguity, so+ # a trailing slash must not turn it into a slug-only link.+ it "still refuses an overlong digit run that ends in a slash" do+ raw = "see https://forum.example.com/t/77777777777777777789999/ there"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ expect(extractor.engine_refusals).to eq(invalid_internal_route: 1)+ end++ {+ "a `.json` topic link" => ["/t/some-slug/18978.json", 18_978, ".json"],+ "an `.rss` topic link" => ["/t/some-slug/233865.rss", 233_865, ".rss"],+ "a `.json` id-only topic link" => ["/t/18978.json", 18_978, ".json"],+ }.each do |label, (path, id, suffix)|+ it "reads #{label}, keeping the format in the suffix" do+ link, = link_for("[x](#{path})")++ expect(link).to include(+ target_type: enums::LinkTarget::TOPIC,+ target_id: id,+ target_suffix: suffix,+ )+ end+ end++ it "reads a `.json` post link" do+ link, = link_for("[x](/p/55.json)")++ expect(link).to include(+ target_type: enums::LinkTarget::POST,+ target_id: 55,+ target_suffix: ".json",+ )+ end++ # Without the format boundary the id folds into the slug path and the route+ # names a category that never existed.+ it "reads a `.json` category link by its id" do+ link, = link_for("[x](/c/support/6.json)")++ expect(link).to include(+ target_type: enums::LinkTarget::CATEGORY,+ target_id: 6,+ target_name: nil,+ target_suffix: ".json",+ )+ end++ it "reads a format extension after a post number" do+ link, = link_for("[x](/t/some-slug/18978/4.json)")++ expect(link).to include(+ target_type: enums::LinkTarget::POST,+ target_topic_id: 18_978,+ target_post_number: 4,+ target_suffix: ".json",+ )+ end++ it "keeps a query behind the format extension in the suffix" do+ raw = "https://forum.example.com/t/some-slug/18978.json?filter_top_level_replies=true"+ link, result = link_for(raw)++ expect(link).to include(+ url: raw,+ target_type: enums::LinkTarget::TOPIC,+ target_id: 18_978,+ target_suffix: ".json?filter_top_level_replies=true",+ original_markdown: raw,+ url_offset: 0,+ )+ expect(result).to eq(link[:placeholder])+ end++ # Only letters make a format extension, so a dotted number after the id+ # stays the ambiguity it is and refuses.+ it "refuses a dotted number after the topic id" do+ raw = "see https://forum.example.com/t/some-slug/18978.5 there"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ expect(extractor.engine_refusals).to eq(invalid_internal_route: 1)+ end++ it "still reads the id of a topic whose slug holds a dot" do+ link, = link_for("[x](/t/rails-7.1-upgrade/18978)")++ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 18_978)+ end+ end++ describe "coordinate-free family pages" do+ let(:internal_link_hosts) { { source_host => nil } }++ it "records a bare family index page as a site link" do+ link, result = link_for("see https://#{source_host}/u/ now")++ expect(link).to include(+ target_type: enums::LinkTarget::SITE,+ url: "https://#{source_host}/u/",+ )+ expect(result).to eq("see #{link[:placeholder]} now")+ end++ it "records a family index page with a query as a site link" do+ link, = link_for("[badges](https://#{source_host}/badges/?_escaped_fragment_=1)")++ expect(link).to include(target_type: enums::LinkTarget::SITE)+ end++ it "keeps an empty slug in front of an id a refusal" do+ raw = "[x](https://#{source_host}/t//209)"++ expect(extract(raw)).to eq(raw)+ expect(extractor.engine_refusals).to eq(invalid_internal_route: 1)+ end++ it "takes a bare trailing dot after the id as an empty format" do+ link, = link_for("[x](https://#{source_host}/t/using-object-storage/148916.)")++ expect(link).to include(+ target_type: enums::LinkTarget::TOPIC,+ target_id: 148_916,+ target_suffix: ".",+ )+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_site_targets_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_site_targets_spec.rbnew file mode 100644index 00000000000..761d0bf1122--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_site_targets_spec.rb@@ -0,0 +1,176 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ # An absolute URL on the source's own host whose path parses no route is still+ # rewritten: only its origin moves to the destination, the rest rides in the+ # suffix. A relative route-less URL stays literal (it's domain-free already).+ describe "SITE targets on a root install" do+ let(:internal_link_hosts) { { source_host => nil } }++ # A coordinate-shaped path that parses no route plausibly carries the OLD+ # site's ids — `/t//209` is an empty-slug spelling of topic 209 — so an+ # origin-only rewrite would point the new host at stale coordinates. Worse+ # than verbatim: the construct stays untouched and the body is reported.+ {+ "an empty topic slug" => "https://forum.example.com/t//209",+ "a junk topic id" => "https://forum.example.com/t/slug/5a",+ "an invalid user segment" => "https://forum.example.com/u/bob!!!",+ # Core's canonical route order would read the trailing number as a tag id,+ # the legacy order as more slug path — the parser refuses the ambiguity+ # instead of guessing a reading.+ "a multi-tag form with a trailing numeric segment" =>+ "https://forum.example.com/tags/c/support/feature/12",+ "a slug followed by a non-id segment" => "https://forum.example.com/t/a-topic-slug/extra!",+ }.each do |label, url|+ it "refuses #{label} instead of recording a SITE link" do+ raw = "This topic you've named [>>>>>](#{url}) doesn't load"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ expect(extractor.engine_refusals).to eq(invalid_internal_route: 1)+ end+ end++ # The bare family segment is that family's index page — coordinate-free,+ # so it keeps the origin rewrite like any other site page.+ it "records a bare route-family index page as a SITE link" do+ link, = link_for("the user directory https://forum.example.com/u lists everyone")++ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: "/u")+ end++ it "does not read a family segment inside a longer word" do+ link, = link_for("[the team](https://forum.example.com/team)")++ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: "/team")+ end++ it "records a real route-less page as a SITE link" do+ link, = link_for("[faq](https://forum.example.com/faq)")++ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: "/faq")+ end++ it "keeps a query string in a SITE link's suffix" do+ link, = link_for("[search](https://forum.example.com/search?q=cats)")++ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: "/search?q=cats")+ end++ it "keeps a fragment in a SITE link's suffix" do+ link, = link_for("[about](https://forum.example.com/about#team)")++ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: "/about#team")+ end++ it "leaves a relative route-less URL literal" do+ raw = "[faq](/faq)"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ end++ # A bare URL on the source's own host is a link once cooked, whatever its+ # path, so it gets the same origin rewrite the bracketed form does. Before,+ # only a path carrying a route segment was recognized bare, and a plain+ # `/faq` in prose survived the migration pointing at the dead origin.+ {+ "a route-less page" => %w[https://forum.example.com/faq /faq],+ "a page mid-sentence" => ["see https://forum.example.com/faq here", "/faq"],+ "a query string" => %w[https://forum.example.com/search?q=cats /search?q=cats],+ "a fragment" => %w[https://forum.example.com/about#team /about#team],+ "a protocol-relative URL" => %w[//forum.example.com/faq /faq],+ "the site root" => %w[https://forum.example.com/ /],+ # Route-less on purpose: with no route segment the host is the only thing+ # that gets this body past the presence gate, so an upper-cased one only+ # works if that alternative is genuinely case-insensitive.+ "an upper-cased host and no route" => %w[HTTPS://FORUM.EXAMPLE.COM/faq /faq],+ "a mixed-case host and no route" => %w[https://FORUM.example.com/faq /faq],+ }.each do |label, (raw, suffix)|+ it "records a bare URL with #{label} as a SITE link" do+ link, = link_for(raw)++ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: suffix)+ end+ end++ # A URL with no path is the forum's own front page, and its origin needs+ # rewriting like any other — otherwise it survives pointing at the site being+ # retired. A query or fragment with no path is the same page.+ {+ "the front page" => ["https://forum.example.com", nil],+ "the front page in link syntax" => ["[home](https://forum.example.com)", nil],+ "the front page mid-sentence" => ["see https://forum.example.com here", nil],+ "a query with no path" => %w[https://forum.example.com?ref=x&a=1 ?ref=x&a=1],+ "a fragment with no path" => %w[https://forum.example.com#top #top],+ "a protocol-relative front page" => ["//forum.example.com", nil],+ "a front page with a default port" => ["https://forum.example.com:443", nil],+ }.each do |label, (raw, suffix)|+ it "records #{label} as a SITE link" do+ link, = link_for(raw)++ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: suffix)+ end+ end++ # `forum.example.com:3000` can be a different application than the forum+ # on `forum.example.com`, so a non-default port is part of the host+ # identity: it matches only a host configured with that port.+ it "leaves a front page on a non-default port literal" do+ link, result = link_for("https://forum.example.com:3000")++ expect(link).to be_nil+ expect(result).to eq("https://forum.example.com:3000")+ end++ it "records a front page on a non-default port when the host is configured with it" do+ extractor =+ described_class.new(+ embeds: buffer,+ markdown_engine:,+ mention_names:,+ hashtag_names:,+ internal_link_hosts: {+ "forum.example.com:3000" => nil,+ },+ )+ extractor.extract("https://forum.example.com:3000")++ expect(buffer.links.first).to include(target_type: enums::LinkTarget::SITE)+ end++ # The host stops at `?`, so the query is the path here and not part of a host+ # named `forum.example.com?ref=x`.+ it "keeps a sentence's punctuation out of a front-page URL" do+ link, result = link_for("see https://forum.example.com. Thanks")++ expect(link).to include(url: "https://forum.example.com", target_suffix: nil)+ expect(result).to eq("see #{link[:placeholder]}. Thanks")+ end++ it "still leaves a bare URL on a foreign host alone" do+ raw = "read https://other.example.org/faq here"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ end++ it "leaves a bare front-page URL on a foreign host alone" do+ raw = "read https://other.example.org here"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ end++ # Nothing in the body names the source's host or a route, so the post never+ # reaches the walk — the gate has to stay that selective.+ it "leaves a body of foreign URLs untouched" do+ raw = "See https://other.example.org/x and https://elsewhere.net/y please."++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_spec.rbnew file mode 100644index 00000000000..5476f6c14c5--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_spec.rb@@ -0,0 +1,803 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ describe "internal links" do+ let(:internal_link_hosts) { { source_host => nil } }++ # A digit run past 18 characters overflows the signed 64-bit integers ids are+ # stored in — and names no real record: it's a numeric topic title, a shape a+ # real forum turned out to have, which crashed the insert.+ context "with a digit run too long to be an id" do+ # A route-less numeric-title URL parses no id, and nothing can tell a+ # slug-only title apart from malformed coordinates without guessing — a+ # `/t/…` path that parses no route refuses rather than host-swapping+ # (see `RouteParser.coordinate_shaped?`). No row is created, so the+ # overflowing digit run still never reaches an integer bind.+ it "refuses a numeric-title topic URL rather than guessing at its shape" do+ raw = "this one - https://forum.example.com/t/77777777777777777789999/ fails"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ expect(extractor.engine_refusals).to eq(invalid_internal_route: 1)+ end++ it "leaves an oversized /p/ id as literal text" do+ raw = "see /p/99999999999999999999999 there"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ end++ it "reads an oversized trailing category segment as a slug, not an id" do+ link, _result = link_for("[cat](/c/77777777777777777789999)")++ expect(link).to include(target_id: nil, target_name: "77777777777777777789999")+ end++ it "still defers an 18-digit id" do+ link, _result = link_for("[topic](/t/123456789012345678)")++ expect(link).to include(target_id: 123_456_789_012_345_678)+ end++ it "degrades a quote with an oversized post: number to username-only" do+ extract(%([quote="bob, post:99999999999999999999, topic:5"]\nx\n[/quote]))++ expect(buffer.quotes.first).to include(+ quoted_username: "bob",+ quoted_post_number: nil,+ quoted_topic_id: nil,+ )+ end+ end++ it "defers a topic link with a slug and id" do+ link, = link_for("[the topic](/t/some-slug/123)")++ expect(link).to include(+ url: "/t/some-slug/123",+ target_type: enums::LinkTarget::TOPIC,+ target_id: 123,+ target_suffix: nil,+ )+ end++ # Core routes `/t/<slug>` to the topic carrying that slug, so the slug is+ # the coordinate; resolution looks it up and restores the source unless+ # exactly one topic matches.+ context "with a slug-only topic link" do+ it "defers the topic by its slug" do+ link, = link_for("[topic](/t/how-to-fix-it)")++ expect(link).to include(+ target_type: enums::LinkTarget::TOPIC,+ target_id: nil,+ target_name: "how-to-fix-it",+ target_suffix: nil,+ )+ end++ it "keeps the query tail as suffix" do+ link, = link_for("see https://forum.example.com/t/how-to-fix-it?page=2 there")++ expect(link).to include(target_name: "how-to-fix-it", target_suffix: "?page=2")+ end+ end++ # A destination can carry a title, padding, or angle brackets and still be a+ # link core resolves. Missing those left such a link pointing at the source+ # site after the migration. Each of these cooks to href="/t/slug/5".+ {+ "a title" => %([t](/t/slug/5 "title")),+ "a single-quoted title" => "[t](/t/slug/5 'title')",+ "a parenthesized title" => "[t](/t/slug/5 (title))",+ "padding" => "[t]( /t/slug/5 )",+ "angle brackets" => "[t](</t/slug/5>)",+ "angle brackets and a title" => %([t](</t/slug/5> "ti")),+ "an absolute URL and a title" => %([t](https://forum.example.com/t/slug/5 "ti")),+ }.each do |label, raw|+ it "defers a topic link written with #{label}" do+ link, = link_for(raw)++ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 5)+ end+ end++ # Core builds no link for either of these, so neither may be rewritten.+ ["[t](/t/slug/5 not-a-title)", %([t](/t/slug/5 "unclosed))].each do |raw|+ it "leaves #{raw} literal, since core makes no link of it" do+ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ end+ end++ it "defers the id-only topic form" do+ link, = link_for("[x](/t/123)")++ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 123)+ end++ it "defers the slugless `/t/-/<id>` topic form" do+ link, = link_for("[x](/t/-/77)")++ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 77)+ end++ it "defers a post link by coordinates, recording no target_id" do+ link, = link_for("[x](/t/some-slug/123/4)")++ expect(link).to include(+ target_type: enums::LinkTarget::POST,+ target_id: nil,+ target_topic_id: 123,+ target_post_number: 4,+ )+ end++ it "defers the slugless post-coordinates form" do+ link, = link_for("[x](/t/12/3)")++ expect(link).to include(+ target_type: enums::LinkTarget::POST,+ target_topic_id: 12,+ target_post_number: 3,+ )+ end++ it "defers a `/p/<id>` post link" do+ link, = link_for("[x](/p/55)")++ expect(link).to include(+ target_type: enums::LinkTarget::POST,+ target_id: 55,+ target_topic_id: nil,+ )+ end++ it "defers a user link by name, for both `/u/` and `/users/`" do+ expect(link_for("[x](/u/bob)").first).to include(+ target_type: enums::LinkTarget::USER,+ target_name: "bob",+ )++ buffer.clear+ expect(link_for("[x](/users/alice)").first).to include(+ target_type: enums::LinkTarget::USER,+ target_name: "alice",+ )+ end++ it "defers a category link by id when the path ends in a number" do+ link, = link_for("[x](/c/support/billing/6)")++ expect(link).to include(+ target_type: enums::LinkTarget::CATEGORY,+ target_id: 6,+ target_name: nil,+ )+ end++ it "defers a category link with no slug at all" do+ link, = link_for("[x](/c/6)")++ expect(link).to include(+ target_type: enums::LinkTarget::CATEGORY,+ target_id: 6,+ target_name: nil,+ )+ end++ # `/c/<slug>/<id>/l/latest` is what a category's Latest tab links to, so the+ # route has to stop at the id and let the tail ride in the suffix — otherwise+ # the tail reads as more slug and the category never resolves.+ it "keeps a category's filter tail out of the slug path" do+ link, = link_for("[x](/c/support/6/l/latest)")++ expect(link).to include(+ target_type: enums::LinkTarget::CATEGORY,+ target_id: 6,+ target_name: nil,+ target_suffix: "/l/latest",+ )+ end++ it "keeps a filter tail out of a parent/child category path" do+ link, = link_for("[x](/c/parent/child/6/l/top)")++ expect(link).to include(target_id: 6, target_name: nil, target_suffix: "/l/top")+ end++ # Categories nest three deep, so the slug path has to survive a third level+ # with the tab tail still coming off it.+ {+ "no tail" => ["/c/gp/parent/child", "gp:parent:child", nil],+ "a filter tail" => %w[/c/gp/parent/child/l/latest gp:parent:child /l/latest],+ "a period tail" => %w[/c/gp/parent/child/l/top/weekly gp:parent:child /l/top/weekly],+ "a /none tail" => %w[/c/gp/parent/child/none gp:parent:child /none],+ }.each do |label, (path, name, suffix)|+ it "reads a three-level category slug path with #{label}" do+ link, = link_for("[x](#{path})")++ expect(link).to include(target_name: name, target_id: nil, target_suffix: suffix)+ end+ end++ it "reads a three-level category by id, with the tail in the suffix" do+ link, = link_for("[x](/c/gp/parent/child/6/l/latest)")++ expect(link).to include(target_id: 6, target_name: nil, target_suffix: "/l/latest")+ end++ # A tag name is a single segment, so anything after it is already suffix —+ # the tab tails that needed guarding on the category routes cost nothing here.+ {+ "a filter tail" => %w[/tag/release/l/latest /l/latest],+ "a period tail" => %w[/tags/release/l/top/weekly /l/top/weekly],+ "a /none tail" => %w[/tag/release/none /none],+ }.each do |label, (path, suffix)|+ it "keeps #{label} out of a tag name" do+ link, = link_for("[x](#{path})")++ expect(link).to include(+ target_type: enums::LinkTarget::TAG,+ target_name: "release",+ target_suffix: suffix,+ )+ end+ end++ it "keeps a filter tail out of a slugless category link" do+ link, = link_for("[x](/c/6/l/latest)")++ expect(link).to include(target_id: 6, target_name: nil, target_suffix: "/l/latest")+ end++ it "reads the trailing id when a category slug is itself numeric" do+ link, = link_for("[x](/c/2015/6)")++ expect(link).to include(target_id: 6, target_name: nil)+ end++ it "reads the trailing category id after a numeric subcategory slug" do+ link, = link_for("[x](/c/parent/123/456/l/latest)")++ expect(link).to include(target_id: 456, target_name: nil, target_suffix: "/l/latest")+ end++ it "keeps a category link's query string in the suffix" do+ link, = link_for("[x](/c/support/6?ascending=false)")++ expect(link).to include(target_id: 6, target_suffix: "?ascending=false")+ end++ it "defers a legacy category link by its parent:child slug path" do+ link, = link_for("[x](/c/support/billing)")++ expect(link).to include(+ target_type: enums::LinkTarget::CATEGORY,+ target_id: nil,+ target_name: "support:billing",+ )+ end++ it "defers a category link using the legacy `/category/` route" do+ link, = link_for("[x](/category/support/billing)")++ expect(link).to include(+ target_type: enums::LinkTarget::CATEGORY,+ target_id: nil,+ target_name: "support:billing",+ )+ end++ # Core routes every tab (`/l/<filter>`, `/none`, `/all`, `/subcategories`)+ # inside `c/*category_slug_path_with_id`, so Rails strips the tail before+ # `Category.find_by_slug_path_with_id` sees the glob. The id-less form has to+ # stop there too, or the tab name folds into the slug and names no category.+ describe "filter tails on the legacy id-less form" do+ {+ "/c/support/l/latest" => %w[support /l/latest],+ "/c/support/l/top/weekly" => %w[support /l/top/weekly],+ "/c/parent/child/l/hot" => %w[parent:child /l/hot],+ "/c/support/none" => %w[support /none],+ "/c/support/all" => %w[support /all],+ "/c/support/subcategories" => %w[support /subcategories],+ }.each do |url, (name, suffix)|+ it "reads #{url} as #{name.inspect} with the tail in the suffix" do+ link, = link_for("[x](#{url})")++ expect(link).to include(+ target_type: enums::LinkTarget::CATEGORY,+ target_id: nil,+ target_name: name,+ target_suffix: suffix,+ )+ end+ end++ # Only `/l/<filter>` is a tab; a bare segment that happens to spell a filter+ # is an ordinary subcategory slug, so it stays part of the path.+ it "keeps a subcategory slug that merely spells a filter in the path" do+ link, = link_for("[x](/c/support/latest)")++ expect(link).to include(target_name: "support:latest", target_suffix: nil)+ end++ # `/c/<slug>/none/<tag-slug>/<tag-id>` is a category+tag intersection. The+ # trailing number is the tag's id, so the slug path must not run through the+ # tail and read it as the category's.+ it "does not read an intersection URL's tag id as the category id" do+ link, = link_for("[x](/c/support/none/bug/12)")++ expect(link).to include(+ target_type: enums::LinkTarget::CATEGORY,+ target_id: nil,+ target_name: "support",+ target_suffix: "/none/bug/12",+ )+ end+ end++ it "defers a tag link for both `/tag/` and `/tags/`" do+ expect(link_for("[x](/tag/release)").first).to include(+ target_type: enums::LinkTarget::TAG,+ target_name: "release",+ )++ buffer.clear+ expect(link_for("[x](/tags/release)").first).to include(+ target_type: enums::LinkTarget::TAG,+ target_name: "release",+ )+ end++ it "defers a canonical tag link by id, keeping only the filter tail as its suffix" do+ link, = link_for("[x](/tag/release/42/l/latest)")++ expect(link).to include(+ target_type: enums::LinkTarget::TAG,+ target_id: 42,+ target_name: nil,+ target_suffix: "/l/latest",+ )+ end++ # The multi-tag routes name several records at once, so the reference+ # carries the category the way plain category links do plus the tag path in+ # `target_tag_path` — the importer rebuilds the route only when every+ # coordinate maps.+ describe "multi-tag routes" do+ it "defers a category+tag link with the category addressed by id" do+ link, = link_for("[x](/tags/c/plugin/22/official)")++ expect(link).to include(+ target_type: enums::LinkTarget::CATEGORY_TAG,+ target_id: 22,+ target_name: nil,+ target_tag_path: "official",+ target_suffix: nil,+ )+ end++ it "defers the legacy id-less form with the joined slug path" do+ link, = link_for("[x](/tags/c/food/wine)")++ expect(link).to include(+ target_type: enums::LinkTarget::CATEGORY_TAG,+ target_id: nil,+ target_name: "food",+ target_tag_path: "wine",+ )+ end++ it "reads a multi-level slug path the way category links do" do+ link, = link_for("[x](/tags/c/howto/devs/import)")++ expect(link).to include(target_name: "howto:devs", target_tag_path: "import")+ end++ it "keeps the `none`/`all` subcategory filter in the tag path" do+ link, = link_for("[x](/tags/c/theme/61/none/extra)")++ expect(link).to include(target_id: 61, target_tag_path: "none/extra")+ end++ it "leaves a list-filter tab and the query in the suffix" do+ link, = link_for("[x](/tags/c/plugin/22/official/l/top?period=yearly)")++ expect(link).to include(+ target_id: 22,+ target_tag_path: "official",+ target_suffix: "/l/top?period=yearly",+ )+ end++ it "defers an intersection link with every tag in the path" do+ link, = link_for("[x](/tags/intersection/food/wine/cheese)")++ expect(link).to include(+ target_type: enums::LinkTarget::TAG_INTERSECTION,+ target_id: nil,+ target_name: nil,+ target_tag_path: "food/wine/cheese",+ )+ end+ end++ it "still reads a bare /tags/intersection as a tag with that name" do+ link, = link_for("[x](/tags/intersection)")++ expect(link).to include(target_type: enums::LinkTarget::TAG, target_name: "intersection")+ end++ it "defers a group link by name for canonical and legacy route segments" do+ %w[/g/team /groups/team /group/team].each do |path|+ link, = link_for("[x](#{path})")++ expect(link).to include(target_type: enums::LinkTarget::GROUP, target_name: "team")+ buffer.clear+ end+ end++ it "defers a badge link by id" do+ link, = link_for("[x](/badges/9/great)")++ expect(link).to include(target_type: enums::LinkTarget::BADGE, target_id: 9)+ end++ it "recognizes an absolute link on a configured host" do+ link, = link_for("read https://forum.example.com/t/slug/99 now")++ expect(link).to include(+ url: "https://forum.example.com/t/slug/99",+ target_type: enums::LinkTarget::TOPIC,+ target_id: 99,+ )+ end++ it "recognizes a protocol-relative link on a configured host" do+ link, = link_for("//forum.example.com/t/slug/99")++ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 99)+ end++ # linkify-it reads the scheme case-insensitively, so core links these too;+ # older forums carried over uppercase schemes.+ %w[HTTPS Https HTTP hTTp].each do |scheme|+ it "recognizes a bare link with a #{scheme} scheme" do+ link, = link_for("read #{scheme}://forum.example.com/t/slug/99 now")++ expect(link).to include(+ url: "#{scheme}://forum.example.com/t/slug/99",+ target_type: enums::LinkTarget::TOPIC,+ target_id: 99,+ )+ end++ it "recognizes a markdown link with a #{scheme} scheme" do+ link, = link_for("[the topic](#{scheme}://forum.example.com/t/slug/99)")++ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 99)+ end+ end++ it "recognizes an uppercase scheme with an uppercase host" do+ link, = link_for("read HTTPS://FORUM.EXAMPLE.COM/t/slug/99 now")++ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 99)+ end++ it "leaves an absolute link on a foreign host literal" do+ raw = "elsewhere https://other.example.com/t/slug/99 done"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ end++ it "captures the link text of a markdown link" do+ link, result = link_for("[the topic](/t/slug/12)")++ expect(link).to include(+ text: "the topic",+ target_type: enums::LinkTarget::TOPIC,+ target_id: 12,+ )+ expect(result).to eq(link[:placeholder])+ end++ # Core allows one level of balanced brackets in link text, so a+ # citation-style link is still a link and must be rewritten.+ it "captures link text holding a balanced bracket pair" do+ link, result = link_for("[see [1]](/t/slug/12)")++ expect(link).to include(text: "see [1]", target_id: 12)+ expect(result).to eq(link[:placeholder])+ end++ it "captures link text that is only a bracket pair" do+ link, = link_for("[[1]](/t/slug/12)")++ expect(link).to include(text: "[1]", target_id: 12)+ end++ # `[](url)` is valid CommonMark that core renders as a link with no text —+ # a shape real corpora carry (pasted HTML digests). The empty label+ # records no label span (there is nothing to rewrite in it), and the+ # destination span points at the url as usual.+ it "defers a link with an empty label" do+ link, result = link_for("[](https://forum.example.com/t/slug/12)")++ expect(link).to include(+ text: "",+ target_type: enums::LinkTarget::TOPIC,+ target_id: 12,+ url_offset: 3,+ label_url_offset: nil,+ original_markdown: "[](https://forum.example.com/t/slug/12)",+ )+ expect(result).to eq(link[:placeholder])+ expect(extractor.engine_refusals).to be_empty+ end++ # Links don't nest in core: the inner `[…](…)` wins and the outer bracket+ # stays literal, so the outer text must not match around it.+ it "defers the inner link when the text holds a nested link" do+ link, = link_for("[see [1](/t/nine/9)](/t/slug/12)")++ expect(link).to include(text: "1", target_id: 9)+ end++ it "keeps a bare URL bare (no captured text)" do+ link, = link_for("https://forum.example.com/t/slug/12")++ expect(link[:text]).to be_nil+ end++ it "keeps trailing sentence punctuation out of a bare URL" do+ link, result = link_for("go to https://forum.example.com/t/slug/12. Thanks")++ expect(link[:url]).to eq("https://forum.example.com/t/slug/12")+ expect(result).to eq("go to #{link[:placeholder]}. Thanks")+ end++ it "captures a trailing sub-path as the suffix" do+ link, = link_for("[x](/u/bob/summary)")++ expect(link).to include(target_name: "bob", target_suffix: "/summary")+ end++ it "captures a query string as the suffix" do+ link, = link_for("[x](/users/alice?u=x)")++ expect(link).to include(target_name: "alice", target_suffix: "?u=x")+ end++ it "captures a fragment as the suffix" do+ link, = link_for("[x](/t/slug/12#reply)")++ expect(link).to include(target_id: 12, target_suffix: "#reply")+ end++ # An image's source moves to the destination site like a link's: the origin+ # is stale, and a route in its path names a record whose id changes. The+ # recorded span is the whole ``, so the importer splices the new+ # destination into it and the image stays an image.+ it "defers an image whose source is an internal route" do+ link, result = link_for("")++ expect(link).to include(+ url: "/t/slug/1",+ text: "pic",+ target_type: enums::LinkTarget::TOPIC,+ target_id: 1,+ original_markdown: "",+ url_offset: 7,+ )+ expect(result).to eq(link[:placeholder])+ end++ it "defers an image whose source is a route-less page on the source's host" do+ link, result = link_for("")++ expect(link).to include(+ url: "https://forum.example.com/tags",+ text: "Tag",+ target_type: enums::LinkTarget::SITE,+ target_suffix: "/tags",+ original_markdown: "",+ url_offset: 7,+ )+ expect(result).to eq(link[:placeholder])+ end++ # An avatar URL, the shape a quoted post's header carries.+ it "defers an image with an empty alt text" do+ raw = ""+ link, result = link_for(raw)++ expect(link).to include(+ text: "",+ target_type: enums::LinkTarget::SITE,+ target_suffix: "/user_avatar/forum.example.com/bob/40/1_1.png",+ original_markdown: raw,+ url_offset: 4,+ )+ expect(result).to eq(link[:placeholder])+ end++ it "leaves an image whose source is on a foreign host alone" do+ raw = ""++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ expect(extractor.engine_refusals).to be_empty+ end++ # CommonMark allows balanced parentheses in a destination.+ it "takes a destination with a parenthesized query whole" do+ raw = "[these](https://forum.example.com/search?q=(oauth2_basic)%20x)"+ link, result = link_for(raw)++ expect(link).to include(+ url: "https://forum.example.com/search?q=(oauth2_basic)%20x",+ text: "these",+ target_type: enums::LinkTarget::SITE,+ target_suffix: "/search?q=(oauth2_basic)%20x",+ original_markdown: raw,+ url_offset: 8,+ )+ expect(result).to eq(link[:placeholder])+ end++ # A space ends the destination, so core forms no link and linkifies the+ # host on its own. The URL is rewritten where it sits and the broken syntax+ # around it stays the author's.+ it "rewrites the linkified host of a destination core rejected" do+ link, result = link_for("[x](https://forum.example.com /categories) tail")++ expect(link).to include(+ url: "https://forum.example.com",+ text: nil,+ target_type: enums::LinkTarget::SITE,+ target_suffix: nil,+ original_markdown: "https://forum.example.com",+ url_offset: 0,+ )+ expect(result).to eq("[x](#{link[:placeholder]} /categories) tail")+ end++ it "rewrites the linkified prefix of a destination whose path holds a space" do+ link, result = link_for("[staff](https://forum.example.com/à propos) checked")++ expect(link).to include(+ url: "https://forum.example.com/à",+ target_type: enums::LinkTarget::SITE,+ target_suffix: "/à",+ original_markdown: "https://forum.example.com/à",+ url_offset: 0,+ )+ expect(result).to eq("[staff](#{link[:placeholder]} propos) checked")+ end++ # Links don't nest, so the emphasis-wrapped image is not a label any link+ # grammar takes. The image is deferred at its own `**](https://forum.example.com/)")++ expect(buffer.uploads.first[:upload_id]).to eq(sha1)+ link = buffer.links.first+ expect(link).to include(+ url: "https://forum.example.com/",+ text: nil,+ target_type: enums::LinkTarget::SITE,+ target_suffix: "/",+ )+ expect(result).to eq("[**#{buffer.uploads.first[:placeholder]}**](#{link[:placeholder]})")+ end++ it "leaves a relative URL inside a prose paren group literal" do+ raw = "(/t/slug/5)"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ end++ it "rewrites an absolute self-host URL inside a prose paren group" do+ link, result = link_for("(https://forum.example.com/t/slug/5)")++ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 5)+ expect(result).to eq("(#{link[:placeholder]})")+ end++ # Core linkifies a bare absolute URL after anything but an ASCII letter, digit+ # or `+` (see `internal_links_parity_spec.rb`), so a URL glued right after+ # prose punctuation is a link once cooked — the construct rewrites it too.+ it "rewrites a bare URL glued to preceding punctuation" do+ link, result = link_for("see,https://forum.example.com/t/slug/5 ok")++ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 5)+ expect(result).to eq("see,#{link[:placeholder]} ok")+ end++ # `_` is admitted too: markdown-it's inline linkify boundary rejects only+ # `[A-Za-z0-9.+-]`, and its core-ruler pass excludes `_` but the inline one+ # does not, so the union linkifies after `_`.+ it "rewrites a bare URL glued to a preceding underscore" do+ link, = link_for("x_https://forum.example.com/t/slug/5 ok")++ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 5)+ end++ # A URL glued right after an ASCII letter isn't linkified by core, and neither+ # is the `//host` inside it a standalone protocol-relative link (linkify-it's+ # `//` schema rejects the `://` tail), so the whole run stays literal.+ it "leaves a bare URL glued to a preceding word character literal" do+ raw = "sitehttps://forum.example.com/t/slug/5"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ end++ # A `\` escapes the following character in markdown, so core forms no link;+ # the construct leaves the URL literal to match.+ it "leaves a backslash-escaped bare URL literal" do+ raw = "see \\https://forum.example.com/t/slug/5 ok"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ end++ it "defers the inner image and rewrites the outer topic URL of a linked image" do+ inner = "https://forum.example.com/uploads/default/original/1X/#{sha1}.png"+ result = extract("[](https://forum.example.com/t/some-topic/5)")++ expect(buffer.uploads.first[:upload_id]).to eq(sha1)+ link = buffer.links.first+ expect(link).to include(+ url: "https://forum.example.com/t/some-topic/5",+ target_type: enums::LinkTarget::TOPIC,+ target_id: 5,+ )+ expect(result).to eq("[#{buffer.uploads.first[:placeholder]}](#{link[:placeholder]})")+ end++ it "rewrites the relative outer topic URL of a linked image" do+ result = extract("[](/t/slug/5)")++ expect(buffer.uploads.first[:upload_id]).to eq(sha1)+ link = buffer.links.first+ expect(link).to include(url: "/t/slug/5", target_type: enums::LinkTarget::TOPIC, target_id: 5)+ expect(result).to eq("[#{buffer.uploads.first[:placeholder]}](#{link[:placeholder]})")+ end++ it "does not extract an internal link inside a fenced code block" do+ raw = <<~MD+ real [x](/t/slug/12)++ ```+ code [x](/t/slug/99) here+ ```+ MD++ result = extract(raw)++ expect(buffer.links.map { |l| l[:target_id] }).to eq([12])+ expect(result).to include("code [x](/t/slug/99) here")+ end++ it "detects a relative link only in link form when no host set is given" do+ plain_extractor =+ described_class.new(embeds: buffer, mention_names:, hashtag_names:, markdown_engine:)++ result = plain_extractor.extract("bare /t/slug/12 and linked [x](/t/slug/34)")++ # With no host set a relative link still qualifies, but only where it is a+ # real link: the bare one in prose stays literal, the link-form one defers.+ expect(buffer.links.map { |l| l[:url] }).to eq(["/t/slug/34"])+ expect(result).to include("bare /t/slug/12 and linked")+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_subdirectory_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_subdirectory_spec.rbnew file mode 100644index 00000000000..35544d9b60a--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_subdirectory_spec.rb@@ -0,0 +1,89 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ describe "subdirectory installs" do+ let(:internal_link_hosts) { { "www.example.com" => "/forum" } }+ let(:internal_link_base_prefix) { "/forum" }++ it "detects a subfolder absolute link, stripping the prefix before the route" do+ link, = link_for("[x](https://www.example.com/forum/t/slug/5)")++ expect(link).to include(+ target_type: enums::LinkTarget::TOPIC,+ target_id: 5,+ target_suffix: nil,+ )+ end++ # The prefix is what belongs to the forum, so `/forum` is the front page here.+ it "records the prefix itself as the front page" do+ link, = link_for("https://www.example.com/forum")++ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: nil)+ end++ # The host's own root is somebody else's app, not the forum, so rewriting its+ # origin would move a link that never pointed at us.+ %w[https://www.example.com https://www.example.com/ https://www.example.com?ref=x].each do |raw|+ it "leaves #{raw} alone, since only the prefix belongs to the forum" do+ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ end+ end++ it "detects a subfolder absolute bare URL in prose" do+ link, = link_for("look https://www.example.com/forum/t/slug/5 now")++ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 5)+ end++ it "detects a relative link carrying the base prefix" do+ link, = link_for("[x](/forum/t/slug/5)")++ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 5)+ end++ it "records a route-less path inside the prefix as a SITE link with the rest" do+ link, = link_for("[faq](https://www.example.com/forum/faq)")++ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: "/faq")+ end++ it "leaves a sibling app's path on the same host literal" do+ raw = "[x](https://www.example.com/other-app/x)"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ end++ it "does not strip the prefix off a host path that only shares its leading text" do+ raw = "[x](https://www.example.com/forumxyz/t/slug/5)"++ expect(extract(raw)).to eq(raw)+ expect(buffer.links).to be_empty+ end++ it "does not fire the foreign-host signal for a sibling app on a prefixed host" do+ foreign_hosts = []+ signalling =+ described_class.new(+ embeds: buffer,+ markdown_engine:,+ mention_names:,+ hashtag_names:,+ internal_link_hosts: {+ "www.example.com" => "/forum",+ },+ internal_link_base_prefix: "/forum",+ on_foreign_host: ->(host) { foreign_hosts << host },+ )++ signalling.extract("[x](https://www.example.com/other-app/t/slug/5)")++ expect(foreign_hosts).to be_empty+ expect(buffer.links).to be_empty+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/mentions_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/mentions_parity_spec.rbnew file mode 100644index 00000000000..2925fac9d87--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/mentions_parity_spec.rb@@ -0,0 +1,120 @@+# frozen_string_literal: true++# Cross-checks the mention construct against what core actually renders. For every+# boundary character in a representative set we build `a<char>@someuser x` (and the+# forward variant `a @someuser<char> x`) and assert the construct extracts exactly+# when `PrettyText.cook` produces a cooked mention link. Mentions run through the+# same text-post-process engine as hashtags, which imposes a whitespace-or-+# punctuation boundary on both sides of the whole match that the rule regex never+# shows, so the boundary is checked here against PrettyText rather than read off+# core's regex. Needs a booted Rails environment, so it is tagged `:rails` and runs+# only under `MIGRATIONS_RAILS=1`.+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do+ include_context "with parity extractor"++ # The construct reads a username the Unicode-aware way (`@café`), which can only+ # be a real source username when the source ran with unicode usernames on, so+ # that is the setting under which the two sides are comparable. With it off core+ # would use an ASCII-only `\w` name and no multibyte username could exist to gate+ # on anyway.+ before do+ SiteSetting.enable_mentions = true+ SiteSetting.unicode_usernames = true+ end++ let!(:user) { Fabricate(:user, username: "someuser") }++ # Extraction is gated on the source's names, so the construct defers only a+ # mention that names something real — the same condition under which core cooks a+ # link (a name that resolves to nothing cooks an inert `<span class="mention">`).+ let(:mention_names) do+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("someuser")])+ end++ def construct_extracts?(raw)+ buffer = new_buffer+ build_extractor(buffer).extract(raw)+ buffer.mentions.any?+ end++ # A real rendered mention is an anchor link. Core cooks an inert+ # `<span class="mention">` (no link) for a name that resolves to nothing, so the+ # anchor is what tells apart a mention core actually rendered.+ def core_cooks?(raw)+ PrettyText.cook(raw, user_id: user.id).include?('<a class="mention"')+ end++ def deviations_for(direction)+ BoundaryCorpus.chars.filter_map do |label, char|+ raw = direction == :before ? "a#{char}@someuser x" : "a @someuser#{char} x"+ extracted = construct_extracts?(raw)+ cooked = core_cooks?(raw)+ next if extracted == cooked++ "#{direction} #{label} #{BoundaryCorpus.describe(char)}: construct=#{extracted} core=#{cooked}"+ end+ end++ it "extracts exactly when core cooks, for every character before the `@`" do+ deviations = deviations_for(:before)+ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end++ it "extracts exactly when core cooks, for every character after the name" do+ deviations = deviations_for(:forward)+ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end++ it "extracts a mention at the very start of the input, matching core" do+ raw = "@someuser x"+ expect(construct_extracts?(raw)).to eq(core_cooks?(raw))+ end++ describe "name shapes" do+ def extracted_name(raw, name)+ buffer = new_buffer+ names = Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize(name)])+ build_extractor(buffer, mention_names: names).extract(raw)+ buffer.mentions.first&.[](:name)+ end++ # The name core linked, read back from the `/u/<slug>` href (URL-encoded), so+ # `@café` links `caf%C3%A9`.+ def cooked_link(raw)+ slug = PrettyText.cook(raw, user_id: user.id)[%r{href="/u/([^"]+)"}, 1]+ slug && CGI.unescape(slug)+ end++ [+ ["john.doe", "hi @john.doe there", "john.doe"], # interior dot+ ["john", "thanks @john.", "john"], # trailing dot dropped+ ["j-d", "cc @j-d please", "j-d"], # interior dash+ ["user", "ping @user_ here", "user"], # trailing underscore dropped+ ["café", "cc @café here", "café"], # multibyte+ ].each do |username, raw, expected|+ it "reads #{raw.inspect} the same name core links" do+ Fabricate(:user, username:)+ expect(extracted_name(raw, username)).to eq(expected)+ expect(cooked_link(raw)).to eq(expected)+ end+ end+ end++ # The one deliberate divergence: core's name regex caps a username at 60+ # characters, so a longer `@name` cooks nothing, while the construct has no cap and+ # extracts it. A name that long can't be a real source username (Discourse's own+ # limit is 60), so the gate never defers one — the cap is moot, and we keep the+ # simpler capless construct.+ it "leaves an over-long name literal, matching core's 60-character cap" do+ # The construct grammar itself has no length cap, but the engine's mention+ # rule does — and no token means nothing matches, so extraction inherits+ # core's cap without duplicating it.+ long = "u#{"a" * 60}" # 61 characters+ names = Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize(long)])+ buffer = new_buffer+ build_extractor(buffer, mention_names: names).extract("hi @#{long} x")++ expect(buffer.mentions).to be_empty+ expect(core_cooks?("hi @#{long} x")).to be(false)+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/mentions_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/mentions_spec.rbnew file mode 100644index 00000000000..a1b390551e8--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/mentions_spec.rb@@ -0,0 +1,188 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ describe "mentions" do+ it "defers a mention, recording the username and preserving surrounding text" do+ result = extract("hey @alice, welcome")++ expect(buffer.mentions.size).to eq(1)+ mention = buffer.mentions.first+ expect(mention).to include(mention_type: enums::MentionType::USER, name: "alice")+ expect(result).to eq("hey #{mention[:placeholder]}, welcome")+ end++ context "with a name ending in a capital sigma" do+ # Core resolves the verbatim mention with Ruby's downcase, which maps Σ+ # to σ even at the end of a word.+ let(:mention_names) do+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("Οδυσσευσ")])+ end++ it "extracts the mention whatever case the author wrote it in" do+ result = extract("hey @ΟΔΥΣΣΕΥΣ and `@ΟΔΥΣΣΕΥΣ`")++ expect(buffer.mentions.size).to eq(1)+ expect(buffer.mentions.first[:name]).to eq("ΟΔΥΣΣΕΥΣ")+ expect(result).to eq("hey #{buffer.mentions.first[:placeholder]} and `@ΟΔΥΣΣΕΥΣ`")+ end+ end++ it "defers a mention at the very start of the body" do+ result = extract("@bob hi")++ expect(buffer.mentions.first[:name]).to eq("bob")+ expect(result).to eq("#{buffer.mentions.first[:placeholder]} hi")+ end++ it "does not treat an e-mail address as a mention" do+ raw = "email me at [email protected] please"++ expect(extract(raw)).to eq(raw)+ expect(buffer.mentions).to be_empty+ end++ it "captures a username containing a dot" do+ result = extract("hi @john.doe there")++ expect(buffer.mentions.first[:name]).to eq("john.doe")+ expect(result).to eq("hi #{buffer.mentions.first[:placeholder]} there")+ end++ it "keeps a trailing sentence period out of the name" do+ result = extract("thanks @bob.")++ expect(buffer.mentions.first[:name]).to eq("bob")+ expect(result).to eq("thanks #{buffer.mentions.first[:placeholder]}.")+ end++ it "captures a username with a hyphen" do+ extract("cc @some-user please")++ expect(buffer.mentions.first[:name]).to eq("some-user")+ end++ # Boundary parity with core, verified against PrettyText (the exhaustive battery+ # lives in the `:rails` mentions_parity_spec). The engine that applies core's+ # mentions rule opens a mention only when the characters on both sides of the+ # whole `@name` are whitespace or a punctuation/symbol character.+ it "opens a mention after a punctuation character such as `_`" do+ extract("a_@alice x")++ expect(buffer.mentions.first[:name]).to eq("alice")+ end++ it "defers a mention after an escaping backslash, which core cooks too" do+ # markdown-it strips the escape before the mentions rule runs, so+ # `\\@alice` is a mention on both sides of the migration; the backslash+ # stays where the author put it.+ result = extract("say \\@alice now")++ expect(buffer.mentions.first[:name]).to eq("alice")+ expect(result).to eq("say \\#{buffer.mentions.first[:placeholder]} now")+ end++ it "rejects a trailing character that is neither whitespace nor punctuation" do+ # `²` (superscript two) is category No — not a word character, so the name+ # ends before it, but not a boundary either, so core leaves `@alice²` literal.+ raw = "hi @alice² there"++ expect(extract(raw)).to eq(raw)+ expect(buffer.mentions).to be_empty+ end++ it "classifies mention types via the injected classifier" do+ classifier =+ Migrations::Converters::Discourse::MentionClassifier.new(+ here_mention: "here",+ group_names: %w[admins],+ )+ extractor =+ described_class.new(+ embeds: buffer,+ markdown_engine:,+ mention_names:,+ hashtag_names:,+ mention_classifier: classifier,+ )++ extractor.extract("@gerhard @admins @here all there")++ expect(buffer.mentions.map { |m| [m[:name], m[:mention_type]] }).to eq(+ [+ ["gerhard", enums::MentionType::USER],+ ["admins", enums::MentionType::GROUP],+ ["here", enums::MentionType::HERE],+ ],+ )+ end++ it "records a broadcast mention's name as written, with its verbatim source" do+ # The importer restores that spelling whenever the destination gives it+ # no different here-mention name to remap to.+ classifier = Migrations::Converters::Discourse::MentionClassifier.new(here_mention: "here")+ extractor =+ described_class.new(+ embeds: buffer,+ markdown_engine:,+ mention_names:,+ hashtag_names:,+ mention_classifier: classifier,+ )++ extractor.extract("cc @Here and @ALL")++ expect(+ buffer.mentions.map { |m| m.values_at(:name, :mention_type, :original_markdown) },+ ).to eq(+ [["Here", enums::MentionType::HERE, "@Here"], ["ALL", enums::MentionType::ALL, "@ALL"]],+ )+ end+ end++ describe "the mention name gate" do+ let(:mention_names) do+ Migrations::CompactStringSet.new(+ %w[alice bob john.doe staff here all café_team].map do |name|+ Migrations::NameNormalizer.normalize(name)+ end,+ )+ end++ it "leaves an @word that names nothing on the source as literal text" do+ raw = "meet at @3pm please"++ expect(extract(raw)).to eq(raw)+ expect(buffer.mentions).to be_empty+ end++ it "defers a group mention in the set" do+ extract("cc @staff now")++ expect(buffer.mentions.first[:name]).to eq("staff")+ end++ it "defers the here and all names in the set" do+ extract("@here and @all please")++ expect(buffer.mentions.map { |mention| mention[:name] }).to eq(%w[here all])+ end++ it "matches the set case-insensitively" do+ extract("ping @Bob today")++ expect(buffer.mentions.first[:name]).to eq("Bob")+ end++ it "matches a Unicode name in the set" do+ extract("cc @café_team here")++ expect(buffer.mentions.first[:name]).to eq("café_team")+ end++ it "requires the names, so no caller can defer every @word by accident" do+ expect { described_class.new(embeds: buffer) }.to raise_error(ArgumentError, /mention_names/)+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/quotes_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/quotes_parity_spec.rbnew file mode 100644index 00000000000..a569db5b81c--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/quotes_parity_spec.rb@@ -0,0 +1,199 @@+# frozen_string_literal: true++require "cgi"++# Cross-checks the quote construct's header reading and its trailing-space forward+# check against what core actually renders. Core parses `[quote=…]` through+# discourse-markdown-it's bbcode-block.js block rule, which accepts an unquoted+# header and strips a range of quotation-mark pairs, and only renders the block+# when nothing but spaces or tabs follows the opening tag to the end of its line.+# For each header shape we assert the construct defers exactly when+# `PrettyText.cook` renders a `<aside class="quote">`, and reads back the same+# username core put on it. Needs a booted Rails environment, so it is tagged+# `:rails` and runs only under `MIGRATIONS_RAILS=1`.+#+# The construct deliberately skips core's block-position rules (line start, a real+# `[/quote]` further down, list context) — matching those would need whole-+# document machinery, and over-extracting a `[quote=…]` that core left as raw+# BBCode only renumbers text in place at import. So a handful of rows are known,+# accepted divergences rather than parity; they are pinned separately below so a+# future change in our behavior is noticed.+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do+ include_context "with parity extractor"++ def construct_quote(raw)+ buffer = new_buffer+ build_extractor(buffer).extract(raw)+ buffer.quotes.first+ end++ def construct_extracts?(raw)+ !construct_quote(raw).nil?+ end++ def construct_username(raw)+ construct_quote(raw)&.[](:quoted_username)+ end++ def core_html(raw)+ PrettyText.cook(raw)+ end++ def core_renders_quote?(raw)+ core_html(raw).include?('<aside class="quote')+ end++ # The username core put on the aside, with the HTML entities it escapes (a+ # literal `"` in a mismatched header cooks as `"`) turned back so it+ # compares to the construct's raw header value.+ def core_username(raw)+ escaped = core_html(raw)[/data-username="([^"]*)"/, 1]+ escaped && CGI.unescapeHTML(escaped)+ end++ # Header shapes that are parity: the construct defers exactly when core renders,+ # and reads back the same username. Each `\n` right after the opening tag is the+ # trailing-space forward check's happy path (a bare line end). The username is+ # the value core strips down to, so the mismatched and empty-pair rows keep the+ # quote marks as literal header characters, matching core.+ #+ # label => [raw, expected username]+ def parity_headers+ {+ "unquoted plain" => [%([quote=bob]\nx\n[/quote]), "bob"],+ "unquoted with parts" => [%([quote=bob, post:1, topic:2]\nx\n[/quote]), "bob"],+ "unquoted leading space" => [%([quote= bob]\nx\n[/quote]), "bob"],+ "straight double" => [%([quote="bob"]\nx\n[/quote]), "bob"],+ "straight double with parts" => [%([quote="bob, post:1, topic:2"]\nx\n[/quote]), "bob"],+ "straight single" => [%([quote='bob']\nx\n[/quote]), "bob"],+ "curly double" => [%([quote=“bob”]\nx\n[/quote]), "bob"],+ "guillemet" => [%([quote=«bob»]\nx\n[/quote]), "bob"],+ "curly single" => [%([quote=‘bob’]\nx\n[/quote]), "bob"],+ "low double quote" => [%([quote=„bob“]\nx\n[/quote]), "bob"],+ "double right-right" => [%([quote=”bob”]\nx\n[/quote]), "bob"],+ "low single" => [%([quote=‚bob’]\nx\n[/quote]), "bob"],+ "single guillemet" => [%([quote=‹bob›]\nx\n[/quote]), "bob"],+ "explicit username" => [+ %([quote="Bob Jones, post:1, username:bjones"]\nx\n[/quote]),+ "bjones",+ ],+ "explicit username with a comma name" => [+ %([quote="Doe, John, post:1, username:jd"]\nx\n[/quote]),+ "jd",+ ],+ "comma name without explicit username" => [+ %([quote="Doe, John, post:1"]\nx\n[/quote]),+ "Doe",+ ],+ "mismatched double-single" => [%([quote="bob']\nx\n[/quote]), %("bob')],+ "mismatched single-double" => [%([quote='bob"]\nx\n[/quote]), %('bob")],+ "one-sided opening mark" => [%([quote="bob]\nx\n[/quote]), %("bob)],+ "one-sided closing mark" => [%([quote=bob"]\nx\n[/quote]), %(bob")],+ "empty straight double" => [%([quote=""]\nx\n[/quote]), %("")],+ "empty straight single" => [%([quote='']\nx\n[/quote]), "''"],+ "empty curly double" => [%([quote=“”]\nx\n[/quote]), "“”"],+ "trailing spaces only" => [%([quote="bob"] \nx\n[/quote]), "bob"],+ "trailing tab only" => [%([quote="bob"]\t\nx\n[/quote]), "bob"],+ "own paragraph after text" => [%(text\n\n[quote="bob"]\nx\n[/quote]), "bob"],+ "line start after text" => [%(text\n[quote="bob"]\nx\n[/quote]), "bob"],+ }+ end++ # Headers that name nobody: core renders no aside (`[quote=]`, `[quote= ]`) and+ # neither do we.+ def parity_non_quotes+ {+ "empty header" => %([quote=]\nx\n[/quote]),+ "whitespace-only header" => %([quote= ]\nx\n[/quote]),+ "trailing text after tag" => %([quote="bob"] hello\nx\n[/quote]),+ "trailing text unquoted" => %([quote=bob] hello\nx\n[/quote]),+ }+ end++ it "defers exactly when core renders, reading back the same username" do+ deviations =+ parity_headers.filter_map do |label, (raw, expected)|+ construct = construct_username(raw)+ core = core_username(raw)+ next if construct == expected && core == expected++ "#{label}: construct=#{construct.inspect} core=#{core.inspect} expected=#{expected.inspect}"+ end++ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end++ it "leaves headerless and no-username shapes alone, matching core" do+ deviations =+ parity_non_quotes.filter_map do |label, raw|+ next if !construct_extracts?(raw) && !core_renders_quote?(raw)++ "#{label}: construct=#{construct_extracts?(raw)} core=#{core_renders_quote?(raw)}"+ end++ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end++ it "extracts a header at the very start of the input, matching core" do+ raw = %([quote="bob"]\nx\n[/quote])+ expect(construct_extracts?(raw)).to eq(core_renders_quote?(raw))+ expect(construct_username(raw)).to eq(core_username(raw))+ end++ it "reads a comma-containing display name back the way core joins it" do+ # quotes.js splits the header on commas and joins everything before the+ # `post:` part back together as the display name.+ raw = %([quote="Doe, John, post:1, username:jd"]\nx\n[/quote])+ expect(construct_quote(raw)).to include(quoted_username: "jd", quoted_name: "Doe, John")+ expect(core_html(raw)).to include("Doe, John")+ end++ it "treats a CRLF after the tag as a bare line end, matching core" do+ # markdown-it normalizes CR/CRLF to LF before it parses, so core renders a+ # quote whose opening tag is followed only by a CRLF; the forward check reads+ # the `\r` as a line end for the same reason.+ raw = %([quote="bob"]\r\nx\r\n[/quote])+ expect(construct_extracts?(raw)).to be(true)+ expect(core_renders_quote?(raw)).to be(true)+ end++ # Block position comes from the engine's parsed quote tokens, so a tag core+ # leaves as raw BBCode is never extracted — the old line-oriented walk+ # over-extracted every one of these.+ describe "block position through the engine tier" do+ it "leaves a mid-line tag alone, like core" do+ # Not at a line start, so core renders nothing and neither do we.+ raw = %(some text [quote="bob"]\nx\n[/quote])+ expect(construct_extracts?(raw)).to be(false)+ expect(core_renders_quote?(raw)).to be(false)+ end++ it "leaves a tag inside a list item alone, like core" do+ raw = %(- [quote="bob"]\nx\n[/quote])+ expect(construct_extracts?(raw)).to be(false)+ expect(core_renders_quote?(raw)).to be(false)+ end++ it "leaves an unclosed quote alone, like core" do+ raw = %([quote="bob"]\nx\nno closing tag here)+ expect(construct_extracts?(raw)).to be(false)+ expect(core_renders_quote?(raw)).to be(false)+ end++ it "extracts core's single-line inline form through the engine tier" do+ # The line-oriented walk's forward check had to decline this shape; the+ # engine tier takes block context from the parsed quote token, so+ # the header is remapped exactly where core renders it.+ raw = %([quote="bob"]body[/quote])+ expect(construct_extracts?(raw)).to be(true)+ expect(core_renders_quote?(raw)).to be(true)+ end++ it "leaves a headerless [quote] alone where core renders an anonymous aside" do+ # Nothing to remap without a header, so we defer to the importer's re-cook.+ raw = %([quote]\nx\n[/quote])+ expect(construct_extracts?(raw)).to be(false)+ expect(core_renders_quote?(raw)).to be(true)+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/quotes_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/quotes_spec.rbnew file mode 100644index 00000000000..34730a02d37--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/quotes_spec.rb@@ -0,0 +1,158 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ describe "quotes" do+ it "records the source coordinates as integers and never a post_id" do+ result = extract(%([quote="bob, post:12, topic:5"]\nquoted body\n[/quote]))++ expect(buffer.quotes.size).to eq(1)+ quote = buffer.quotes.first+ expect(quote).to include(+ quoted_username: "bob",+ quoted_topic_id: 5,+ quoted_post_number: 12,+ quoted_post_id: nil,+ )+ expect(result).to eq("#{quote[:placeholder]}\nquoted body\n[/quote]")+ end++ it "uses the explicit username: part and keeps the display name" do+ extract(%([quote="Bob Jones, post:1, topic:2, username:bjones"]\nhi\n[/quote]))++ expect(buffer.quotes.first).to include(+ quoted_username: "bjones",+ quoted_name: "Bob Jones",+ quoted_topic_id: 2,+ quoted_post_number: 1,+ )+ end++ it "records no name for a bare leading token that IS the username" do+ # Without an explicit username:, the leading token is the username itself+ # (Discourse omits username: when the display name equals it).+ extract(%([quote="jane, post:1"]\nhi\n[/quote]))++ expect(buffer.quotes.first).to include(quoted_username: "jane", quoted_name: nil)+ end++ it "records no name when the display name equals the explicit username" do+ extract(%([quote="jane, post:1, topic:2, username:jane"]\nhi\n[/quote]))++ expect(buffer.quotes.first).to include(quoted_username: "jane", quoted_name: nil)+ end++ # A display name containing a comma splits into several header parts; core's+ # quotes.js joins everything before the `post:` part back together, so the+ # whole name has to survive here too.+ it "keeps a comma-containing display name whole" do+ extract(%([quote="Doe, John, post:1, topic:2, username:jd"]\nhi\n[/quote]))++ expect(buffer.quotes.first).to include(+ quoted_username: "jd",+ quoted_name: "Doe, John",+ quoted_post_number: 1,+ quoted_topic_id: 2,+ )+ end++ it "keeps a bare comma tail out of the username when username: is omitted" do+ extract(%([quote="Doe, John, post:1"]\nhi\n[/quote]))++ expect(buffer.quotes.first).to include(quoted_username: "Doe", quoted_name: nil)+ end++ it "fills the containing topic id when the header names a post but no topic" do+ extract(%([quote="bob, post:12"]\nbody\n[/quote]), topic_id: 77)++ expect(buffer.quotes.first).to include(quoted_topic_id: 77, quoted_post_number: 12)+ end++ # The importer resolves a quoted post from the pair, so a topic on its own is+ # no use to it. The containing topic id is passed here to show it is not+ # substituted either: half a coordinate is dropped rather than guessed at.+ it "drops both coordinates when the header names a topic but no post" do+ extract(%([quote="bob, topic:5"]\nbody\n[/quote]), topic_id: 77)++ expect(buffer.quotes.first).to include(+ quoted_username: "bob",+ quoted_topic_id: nil,+ quoted_post_number: nil,+ )+ end++ it "records no coordinates for a username-only quote" do+ extract(%([quote="alice"]\nhello\n[/quote]), topic_id: 77)++ expect(buffer.quotes.first).to include(+ quoted_username: "alice",+ quoted_topic_id: nil,+ quoted_post_number: nil,+ quoted_post_id: nil,+ )+ end++ it "reads an unquoted header, with and without parts" do+ extract(%([quote=alice]\nhello\n[/quote]))+ expect(buffer.quotes.first).to include(quoted_username: "alice")++ extract(%([quote=bob, post:3, topic:4]\nhi\n[/quote]))+ expect(buffer.quotes.last).to include(+ quoted_username: "bob",+ quoted_post_number: 3,+ quoted_topic_id: 4,+ )+ end++ it "strips a single-quoted, curly or guillemet-wrapped header down to the name" do+ {+ %([quote='alice']\nx\n[/quote]) => "alice",+ %([quote=“alice”]\nx\n[/quote]) => "alice",+ %([quote=«alice»]\nx\n[/quote]) => "alice",+ %([quote=‘alice’]\nx\n[/quote]) => "alice",+ }.each do |raw, username|+ buffer.quotes.clear+ extract(raw)+ expect(buffer.quotes.first).to include(quoted_username: username)+ end+ end++ it "keeps a mismatched or one-sided quote mark as part of the header" do+ # Core strips only a matching pair; anything else stays a literal character,+ # so the header (and the username we read from it) keeps the mark. Verified+ # against PrettyText.+ {+ %([quote="bob']\nx\n[/quote]) => %("bob'),+ %([quote=bob"]\nx\n[/quote]) => %(bob"),+ %([quote=""]\nx\n[/quote]) => %(""),+ }.each do |raw, username|+ buffer.quotes.clear+ extract(raw)+ expect(buffer.quotes.first).to include(quoted_username: username)+ end+ end++ it "extracts when only spaces or tabs follow the opening tag" do+ extract(%([quote="bob"] \t \nx\n[/quote]))++ expect(buffer.quotes.first).to include(quoted_username: "bob")+ end++ it "leaves the tag literal when non-space text follows it on the line" do+ # Core only renders the block when nothing but spaces/tabs follow the opening+ # tag to the end of its line, so text after the tag makes it plain BBCode.+ raw = %([quote="bob"] and then some\nx\n[/quote])++ expect(extract(raw)).to eq(raw)+ expect(buffer.quotes).to be_empty+ end++ it "leaves a quote with no header alone" do+ raw = "[quote]anonymous[/quote]"++ expect(extract(raw)).to eq(raw)+ expect(buffer.quotes).to be_empty+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/sigma_names_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/sigma_names_spec.rbnew file mode 100644index 00000000000..c1696e5f8d7--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/sigma_names_spec.rb@@ -0,0 +1,48 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ let(:mention_names) { Migrations::CompactStringSet.new(%w[κοσμος κοσμοσ]) }+ let(:raw_hashtag_name_list) { %w[κοσμος κοσμοσ] }++ it "records each sigma username in its original spelling" do+ output = extract("@κοσμος @κοσμοσ @ΚΟΣΜΟΣ")++ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[κοσμος κοσμοσ ΚΟΣΜΟΣ])+ expect(output).to eq(buffer.mentions.map { |row| row[:placeholder] }.join(" "))+ expect(extractor.engine_refusals).to be_empty+ end++ it "preserves shielded sigma spellings while extracting live mentions" do+ output = extract("`@κοσμος` @κοσμοσ @κοσμος")++ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[κοσμοσ κοσμος])+ expect(output).to eq("`@κοσμος` #{buffer.mentions.map { |row| row[:placeholder] }.join(" ")}")+ expect(extractor.engine_refusals).to be_empty+ end++ it "records each hashtag with its original sigma identity" do+ output = extract("#κοσμος #κοσμοσ #ΚΟΣΜΟΣ")++ expect(buffer.hashtags.map { |row| row[:name] }).to eq(%w[κοσμος κοσμοσ ΚΟΣΜΟΣ])+ expect(output).to eq(buffer.hashtags.map { |row| row[:placeholder] }.join(" "))+ expect(extractor.engine_refusals).to be_empty+ end++ context "when only the ordinary sigma identity exists" do+ let(:mention_names) { Migrations::CompactStringSet.new(%w[κοσμοσ]) }+ let(:raw_hashtag_name_list) { %w[κοσμοσ] }++ it "leaves unknown sigma spellings literal in a body with known constructs" do+ output = extract("@κοσμος @κοσμοσ #κοσμος #ΚΟΣΜΟΣ")++ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[κοσμοσ])+ expect(buffer.hashtags.map { |row| row[:name] }).to eq(%w[ΚΟΣΜΟΣ])+ expect(output).to eq(+ "@κοσμος #{buffer.mentions.first[:placeholder]} #κοσμος #{buffer.hashtags.first[:placeholder]}",+ )+ expect(extractor.engine_refusals).to be_empty+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/tag_case_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/tag_case_parity_spec.rbnew file mode 100644index 00000000000..c2d109bb9cb--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/tag_case_parity_spec.rb@@ -0,0 +1,109 @@+# frozen_string_literal: true++# Cross-checks case folding against core, construct by construct. They do not+# agree with each other, and the split is not guessable:+#+# * core's bbcode *block* rules fold case — `[CODE]`, `[QUOTE=…]` render the+# same as their lower-case forms, and so does `<PRE>`;+# * the `|attachment]` marker folds case too, but it only picks the CSS class:+# any markdown link to `upload://` carries `data-orig-href` either way, so+# the upload is real and has to be recorded;+# * the `upload://` scheme itself does NOT fold. `UPLOAD://` cooks a link with+# no `data-orig-href` — nothing the importer could resolve.+#+# Getting one wrong is silent in both directions. Too strict and a construct core+# rendered is never rewritten, so its ids stay the source's; too loose and we+# replace text core showed as-is. Both had happened, which is why every construct+# gets a row rather than only the ones that looked doubtful.+#+# Needs a booted Rails environment, so it is tagged `:rails` and runs only under+# `MIGRATIONS_RAILS=1`.+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do+ include_context "with parity extractor"++ # The sentinel mention only defers when the source has its name.+ let(:mention_names) do+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("alice")])+ end++ # Constructs whose content core treats as code: the sentinel mention must+ # survive cooking as plain text, and we must leave the body alone.+ def code_rows+ {+ "[code]" => "[code]\n@alice\n[/code]",+ "[CODE]" => "[CODE]\n@alice\n[/CODE]",+ "[code] with an upper-case closer" => "[code]\n@alice\n[/CODE]",+ "<pre>" => "<pre>\n@alice\n</pre>",+ "<PRE>" => "<PRE>\n@alice\n</PRE>",+ }+ end++ # Constructs core renders as a quote block, which we must record so its+ # coordinates are remapped.+ def quote_rows+ {+ "[quote=…]" => %([quote="alice, post:1, topic:2"]\nx\n[/quote]),+ "[QUOTE=…]" => %([QUOTE="alice, post:1, topic:2"]\nx\n[/QUOTE]),+ }+ end++ # label => [raw, whether core resolves the upload]+ def upload_rows+ {+ "an image" => ["", true],+ "an image with an upper-case scheme" => ["", false],+ "an attachment" => ["[r.pdf|attachment](upload://abc123.png)", true],+ "an upper-case attachment marker" => ["[r.pdf|ATTACHMENT](upload://abc123.png)", true],+ "an attachment with an upper-case scheme" => [+ "[r.pdf|attachment](UPLOAD://abc123.png)",+ false,+ ],+ }+ end++ def extract(raw)+ buffer = new_buffer+ [buffer, build_extractor(buffer).extract(raw)]+ end++ it "leaves the body of every code form alone, as core does" do+ deviations =+ code_rows.filter_map do |label, raw|+ buffer, result = extract(raw)+ core_kept_it_literal = !PrettyText.cook(raw).include?('class="mention"')+ next if core_kept_it_literal && buffer.mentions.empty? && result == raw++ "#{label}: core_literal=#{core_kept_it_literal} ours_rewrote=#{result != raw}"+ end++ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end++ it "records every quote form core renders as a quote" do+ deviations =+ quote_rows.filter_map do |label, raw|+ buffer, = extract(raw)+ core_quotes = PrettyText.cook(raw).include?('class="quote')+ next if core_quotes && buffer.quotes.size == 1++ "#{label}: core_quotes=#{core_quotes} ours_recorded=#{buffer.quotes.size}"+ end++ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end++ it "records an upload exactly when core resolves one" do+ deviations =+ upload_rows.filter_map do |label, (raw, expected)|+ buffer, = extract(raw)+ cooked = PrettyText.cook(raw)+ core_resolves = cooked.include?("data-orig-src") || cooked.include?("data-orig-href")+ ours = buffer.uploads.any?+ next if core_resolves == expected && ours == expected++ "#{label} (#{raw.inspect}): expected=#{expected} core=#{core_resolves} ours=#{ours}"+ end++ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/uploads_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/uploads_parity_spec.rbnew file mode 100644index 00000000000..d82ba80f6c8--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/uploads_parity_spec.rb@@ -0,0 +1,77 @@+# frozen_string_literal: true++# Cross-checks the full-URL upload construct's bare-URL boundary against what core+# renders. It shares {Base#bare_url_boundary_before?} with the internal-link+# construct, so it rides the same linkify boundary (see+# `internal_links_parity_spec.rb`); this smaller battery confirms an upload URL —+# recognized by its 40-hex sha1, not by a route — admits at the same characters.+# For every boundary character we build `a<char><upload-url> b` (and the forward+# variant) and assert the construct defers exactly when `PrettyText.cook` linkifies+# an anchor for the URL. Needs a booted Rails environment, so it is tagged `:rails`+# and runs only under `MIGRATIONS_RAILS=1`.+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do+ include_context "with parity extractor"++ before { SiteSetting.enable_markdown_linkify = true }++ def url+ sha1 = "0123456789abcdef0123456789abcdef01234567"+ "https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"+ end++ def construct_extracts?(raw)+ buffer = new_buffer+ build_extractor(buffer).extract(raw)+ buffer.uploads.any?+ end++ def core_links?(raw)+ hrefs = PrettyText.cook(raw).scan(/<a\b[^>]*href="([^"]*)"/).flatten+ hrefs.any? { |href| href.start_with?(url) }+ end++ def deviations_for(direction)+ BoundaryCorpus.chars.filter_map do |label, char|+ raw = direction == :before ? "a#{char}#{url} b" : "a #{url}#{char} b"+ extracted = construct_extracts?(raw)+ linkified = core_links?(raw)+ next if extracted == linkified++ "#{direction} #{label} #{BoundaryCorpus.describe(char)}: " \+ "construct=#{extracted} core=#{linkified}"+ end+ end++ it "defers exactly when core linkifies, for every character before the URL" do+ deviations = deviations_for(:before)+ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end++ # The sha1 sits mid-URL, so a trailing character never disturbs recognition and+ # the `\w` tail trims the same trailing punctuation linkify does — both sides+ # keep the URL and drop what follows, so the whole set is parity.+ it "defers exactly when core linkifies, for every character after the URL" do+ deviations = deviations_for(:forward)+ expect(deviations).to be_empty, -> { deviations.join("\n") }+ end++ it "defers at the very start of the input, matching core" do+ raw = "#{url} b"+ expect(construct_extracts?(raw)).to eq(core_links?(raw))+ end++ # The converter decodes `/uploads/short-url/<token>` without core on the+ # load path, so it carries its own base62 decoder. This pins it to core's;+ # a drift here means short-URL rows stop matching their upload rows.+ it "decodes short-URL tokens the way core does" do+ tokens = %w[21 aZ9 Zm9vYmFy 2Yjf3WE4KOQ88YUb4fUMubKB9My zzzzzzzzzzzzzzzzzzzzzzzzzzz 0]+ tokens.each do |token|+ expect(+ Migrations::Converters::Discourse::MarkdownScanner::Constructs::UploadUrl.sha1_from_short_token(+ token,+ ),+ ).to eq(Upload.sha1_from_base62_encoded(token)),+ "token #{token.inspect} decoded differently"+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/uploads_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/uploads_spec.rbnew file mode 100644index 00000000000..f6188212216--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/uploads_spec.rb@@ -0,0 +1,732 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::RawExtractor do+ include_context "with raw extractor"++ describe "external-host marking" do+ let(:internal_link_hosts) { { source_host => nil } }+ let(:sha1) { "a" * 40 }++ # The importer maps a foreign-host row's sha1 only against an explicit+ # allowlist — a colliding basename must not rewrite another site's file —+ # so the extractor has to say which rows are foreign.+ it "marks foreign hosts and leaves the source's own and relative URLs unmarked" do+ extract(+ " " \+ " " \+ "",+ )++ expect(buffer.uploads.map { |row| row[:external_host] }).to eq([nil, "cdn.example.com", nil])+ end++ context "with a subdirectory install" do+ let(:internal_link_hosts) { { source_host => "/forum" } }+ let(:internal_link_base_prefix) { "/forum" }++ # On a subdirectory install the host alone does not make an upload the+ # source's own: `/other/uploads/…` on the same host belongs to a+ # different application, and mapping its sha1 could rewrite that+ # application's file on a hash collision.+ it "marks an absolute sibling-application upload as external" do+ extract(+ " " \+ "",+ )++ expect(buffer.uploads.map { |row| row[:external_host] }).to eq([nil, source_host])+ end++ # A relative sibling path has no host an operator could allowlist, so+ # no row is recorded at all and the source text stays.+ it "leaves a relative sibling-application upload verbatim" do+ raw = ""++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ expect(extractor.engine_refusals).to be_empty+ end++ it "records a relative upload inside the base prefix" do+ extract("")++ expect(buffer.uploads.map { |row| row[:external_host] }).to eq([nil])+ end+ end+ end++ describe "unsupported /uploads/ paths" do+ # A generic `/uploads/` path (WordPress and similar) is not an upload+ # reference: it has neither the `original|optimized/` shape nor the+ # short-URL one. It must not become a candidate, a tracked value, or a+ # refusal.+ it "ignores an external generic uploads path" do+ raw = "see https://blog.example.net/wp-content/uploads/2009/12/image1.jpg here"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ expect(extractor.engine_refusals).to be_empty+ end+ end++ describe "short-URL uploads" do+ # `/uploads/short-url/<token>` carries the base62-encoded sha1 (core's+ # `Upload#short_path`), so the row can carry the decoded 40-hex sha1 and+ # resolve like a long URL.+ let(:decoded) { "#{"0" * 38}7d" }++ it "defers a short-URL image, decoding the token to the sha1" do+ result = extract("")++ expect(buffer.uploads.size).to eq(1)+ expect(buffer.uploads.first[:upload_id]).to eq(decoded)+ expect(result).to include(buffer.uploads.first[:placeholder])+ end++ it "defers a bare absolute short URL" do+ result = extract("file at https://forum.example.com/uploads/short-url/21.jpeg here")++ expect(buffer.uploads.first[:upload_id]).to eq(decoded)+ expect(result).to include(buffer.uploads.first[:placeholder])+ end++ it "defers a relative short URL at a link target" do+ result = extract("[file](/uploads/short-url/21.jpeg)")++ expect(buffer.uploads.first[:upload_id]).to eq(decoded)+ expect(result).to eq(buffer.uploads.first[:placeholder])+ end++ it "leaves a token that is too long for a sha1 alone" do+ raw = "see /uploads/short-url/#{"z" * 27}.png here"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end+ end++ describe "uploads" do+ it "defers an image upload, recording the sha1" do+ result = extract("before  after")++ expect(buffer.uploads.size).to eq(1)+ upload = buffer.uploads.first+ expect(upload[:upload_id]).to eq("abc123XYZ")+ expect(result).to eq("before #{upload[:placeholder]} after")+ end++ it "defers an attachment upload" do+ extract("[report.pdf|attachment](upload://Zm9vYmFy.pdf)")++ expect(buffer.uploads.first[:upload_id]).to eq("Zm9vYmFy")+ end++ it "takes the size Discourse writes after an attachment" do+ result = extract("[report.pdf|attachment](upload://Zm9vYmFy.pdf) (1.2 MB)")++ expect(buffer.uploads.first[:upload_id]).to eq("Zm9vYmFy")+ expect(result).to eq(buffer.uploads.first[:placeholder])+ end++ # Only the sha1 is recorded, so whatever the match covers is re-rendered from+ # the destination's metadata and the rest is dropped. The size group must not+ # reach past the line end, or a parenthesized line of the author's own text+ # disappears from the post.+ {+ "the next line" => "\n(this may take a while)",+ "a line after a blank one" => "\n\n(a caption)",+ }.each do |label, tail|+ it "leaves #{label} alone after an attachment" do+ result = extract("[report.pdf|attachment](upload://Zm9vYmFy.pdf)#{tail}")++ expect(buffer.uploads.first[:upload_id]).to eq("Zm9vYmFy")+ expect(result).to eq("#{buffer.uploads.first[:placeholder]}#{tail}")+ end+ end++ it "records the verbatim source as original_markdown for an upload:// reference" do+ extract("")++ expect(buffer.uploads.first[:original_markdown]).to eq("")+ end++ # Core's link parse fails at the unmatched outer `[` and renders the+ # attachment from the inner bracket, keeping `[foo` literal — so the match+ # must not start one bracket too early and swallow that text.+ it "matches from the inner bracket when an unmatched `[` precedes an attachment" do+ result = extract("[foo[bar|attachment](upload://Zm9vYmFy.pdf)")++ expect(buffer.uploads.size).to eq(1)+ expect(buffer.uploads.first[:upload_id]).to eq("Zm9vYmFy")+ expect(result).to eq("[foo#{buffer.uploads.first[:placeholder]}")+ end+ end++ # `[image|281x500](upload://…)` — a failed image paste whose `!` was lost.+ # Core renders the label literally, pipe and all, as a plain link carrying+ # `data-orig-href`, so the upload is as real as in any other form.+ describe "link-position labels with pipe metadata" do+ it "defers the upload, replacing the whole construct" do+ result = extract("pasted [image|281x500](upload://2Yjf3WE4KOQ88YUb4fUMubKB9My.jpeg) here")++ upload = buffer.uploads.first+ expect(upload[:upload_id]).to eq("2Yjf3WE4KOQ88YUb4fUMubKB9My")+ expect(upload[:original_markdown]).to eq(+ "[image|281x500](upload://2Yjf3WE4KOQ88YUb4fUMubKB9My.jpeg)",+ )+ expect(result).to eq("pasted #{upload[:placeholder]} here")+ expect(extractor.engine_refusals).to be_empty+ end++ it "takes a multi-pipe label and an empty pre-pipe part" do+ extract("[a|b|c](upload://Zm9vYmFy.png) and [|690x388](upload://YmFyYmF6.png)")++ expect(buffer.uploads.map { |row| row[:upload_id] }).to eq(%w[Zm9vYmFy YmFyYmF6])+ end++ # The attachment marker folds case (core's split, see the construct), so an+ # upper-cased one still takes the attachment branch — with its size tail,+ # which the link-label branch has no reason to consume.+ it "reads an upper-cased attachment marker as an attachment, size included" do+ result = extract("[r.pdf|ATTACHMENT](upload://Zm9vYmFy.pdf) (1.2 MB)")++ expect(buffer.uploads.first[:upload_id]).to eq("Zm9vYmFy")+ expect(result).to eq(buffer.uploads.first[:placeholder])+ end++ # Core links a pipe-less `[label](upload://…)` the same way, so the upload+ # is recorded. The engine tier confirms the occurrence before the replace,+ # and a resolution miss restores the verbatim label.+ it "defers a pipe-less link" do+ body = "[plain label](upload://Zm9vYmFy.png)"+ result = extract(body)++ expect(buffer.uploads.first[:upload_id]).to eq("Zm9vYmFy")+ expect(buffer.uploads.first[:original_markdown]).to eq(body)+ expect(result).to eq(buffer.uploads.first[:placeholder])+ expect(extractor.engine_refusals).to be_empty+ end++ it "defers a label with one level of nested brackets" do+ body = "[Juan [John] Hernandez.pdf|attachment](upload://Zm9vYmFy.pdf)"+ result = extract(body)++ expect(buffer.uploads.first[:upload_id]).to eq("Zm9vYmFy")+ expect(buffer.uploads.first[:original_markdown]).to eq(body)+ expect(result).to eq(buffer.uploads.first[:placeholder])+ expect(extractor.engine_refusals).to be_empty+ end+ end++ describe "full-URL uploads" do+ it "defers an image referenced by a root-relative upload URL" do+ url = "/uploads/default/original/2X/a/ab/#{sha1}.png"+ result = extract("before  after")++ upload = buffer.uploads.first+ expect(upload[:upload_id]).to eq(sha1)+ expect(upload[:original_markdown]).to eq("")+ expect(result).to eq("before #{upload[:placeholder]} after")+ end++ it "defers a markdown link to an absolute upload URL" do+ url = "https://forum.example.com/uploads/default/original/2X/a/ab/#{sha1}.pdf"+ extract("[report](#{url})")++ expect(buffer.uploads.first).to include(+ upload_id: sha1,+ original_markdown: "[report](#{url})",+ )+ end++ # One label grammar for every `[label](dest)` construct+ # (`Constructs::Base::LINK_TEXT`): a citation-style label used to leave the+ # upload URL unanchored, so the engine tier refused the post instead of+ # extracting it.+ it "defers a link whose label carries one level of nested brackets" do+ url = "https://forum.example.com/uploads/default/original/1X/#{sha1}.png"+ body = "[see [1]](#{url})"+ result = extract(body)++ upload = buffer.uploads.first+ expect(upload).to include(upload_id: sha1, original_markdown: body)+ expect(result).to eq(upload[:placeholder])+ expect(extractor.engine_refusals).to be_empty+ end++ it "defers an image whose alt text carries one level of nested brackets" do+ url = "https://forum.example.com/uploads/default/original/1X/#{sha1}.png"+ body = "![see [1]](#{url})"+ result = extract(body)++ upload = buffer.uploads.first+ expect(upload).to include(upload_id: sha1, original_markdown: body)+ expect(result).to eq(upload[:placeholder])+ expect(extractor.engine_refusals).to be_empty+ end++ it "reads the sha1 from an optimized image variant" do+ url = "/uploads/default/optimized/2X/a/ab/#{sha1}_2_690x388.png"+ extract("")++ expect(buffer.uploads.first[:upload_id]).to eq(sha1)+ end++ it "recognizes a secure-uploads URL" do+ url = "/secure-uploads/original/2X/a/ab/#{sha1}.png"+ extract("")++ expect(buffer.uploads.first[:upload_id]).to eq(sha1)+ end++ it "recognizes a protocol-relative upload URL" do+ url = "//cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"+ extract("")++ expect(buffer.uploads.first[:upload_id]).to eq(sha1)+ end++ # A destination carrying a title is still a link core resolves, so the upload+ # behind it has to be carried over like any other.+ it "recognizes an upload URL in an image with a title" do+ url = "https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"+ extract(%())++ expect(buffer.uploads.first[:upload_id]).to eq(sha1)+ end++ it "recognizes an upload URL in a link with a title" do+ url = "https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.pdf"+ extract(%([report](#{url} 'a title')))++ expect(buffer.uploads.first[:upload_id]).to eq(sha1)+ end++ it "recognizes an upload URL with padding around it" do+ url = "/uploads/default/original/2X/a/ab/#{sha1}.png"+ extract("")++ expect(buffer.uploads.first[:upload_id]).to eq(sha1)+ end++ # linkify-it reads the scheme case-insensitively, so core links these too.+ %w[HTTPS Https HTTP].each do |scheme|+ it "recognizes an upload URL with a #{scheme} scheme" do+ url = "#{scheme}://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"+ extract("")++ expect(buffer.uploads.first[:upload_id]).to eq(sha1)+ end++ it "recognizes a bare upload URL with a #{scheme} scheme" do+ url = "#{scheme}://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"+ extract("see #{url} here")++ expect(buffer.uploads.first[:upload_id]).to eq(sha1)+ end+ end++ it "keeps a bare URL's trailing sentence punctuation out of the match" do+ url = "https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"+ result = extract("look at #{url}.")++ expect(buffer.uploads.first[:original_markdown]).to eq(url)+ expect(result).to eq("look at #{buffer.uploads.first[:placeholder]}.")+ end++ it "leaves a relative upload URL bare in prose literal" do+ url = "/uploads/default/original/2X/a/ab/#{sha1}.png"+ raw = "see #{url} thanks"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "defers a bare absolute upload URL in prose" do+ url = "https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"+ result = extract("see #{url} thanks")++ expect(buffer.uploads.first).to include(upload_id: sha1, original_markdown: url)+ expect(result).to eq("see #{buffer.uploads.first[:placeholder]} thanks")+ end++ # Core linkifies a bare absolute URL after anything but an ASCII letter, digit+ # or `+` (see `uploads_parity_spec.rb`), so a URL glued right after prose+ # punctuation is a link once cooked — the construct defers it too.+ it "defers a bare upload URL glued to preceding punctuation" do+ url = "https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"+ result = extract("here,#{url} thanks")++ expect(buffer.uploads.first).to include(upload_id: sha1, original_markdown: url)+ expect(result).to eq("here,#{buffer.uploads.first[:placeholder]} thanks")+ end++ # A URL glued right after an ASCII letter isn't linkified by core, and the+ # `//host` inside it isn't a standalone protocol-relative link either+ # (linkify-it's `//` schema rejects the `://` tail), so it stays literal.+ it "leaves a bare upload URL glued to a preceding word character literal" do+ url = "https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"+ raw = "here#{url}"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "ignores a non-upload URL" do+ raw = " and https://example.com/page"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "ignores an uploads URL whose basename is not a 40-hex sha1" do+ raw = ""++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ end++ it "does not extract a full-URL upload inside a fenced code block" do+ url = "/uploads/default/original/2X/a/ab/#{sha1}.png"+ raw = <<~MD+ real ++ ```+ code  and bare #{url}+ ```+ MD++ result = extract(raw)++ expect(buffer.uploads.size).to eq(1)+ expect(result).to include("code  and bare #{url}")+ end++ it "defers only the inner image of a linked image, leaving the outer link literal" do+ inner = "https://forum.example.com/uploads/default/original/1X/#{sha1}.png"+ result = extract("[](https://other.example.com/page)")++ expect(buffer.uploads.size).to eq(1)+ upload = buffer.uploads.first+ expect(upload[:upload_id]).to eq(sha1)+ # The inner image alone is deferred; the mangled half-link the greedy `[…]`+ # class used to produce is gone.+ expect(upload[:original_markdown]).to eq("")+ expect(result).to eq("[#{upload[:placeholder]}](https://other.example.com/page)")+ end++ it "defers only the inner image of an image nested in an image description" do+ inner = "https://forum.example.com/uploads/default/original/1X/#{sha1}.png"+ result = extract("](https://elsewhere.example.com/x.png)")++ expect(buffer.uploads.size).to eq(1)+ upload = buffer.uploads.first+ expect(upload[:upload_id]).to eq(sha1)+ # The inner image alone is deferred; the outer `` stay literal+ # instead of the greedy alt class swallowing through the inner `)`.+ expect(upload[:original_markdown]).to eq("")+ expect(result).to eq("![#{upload[:placeholder]}](https://elsewhere.example.com/x.png)")+ end++ it "defers only the inner short-form upload of an image nested in an image description" do+ result = extract("](https://elsewhere.example.com/x.png)")++ expect(buffer.uploads.size).to eq(1)+ upload = buffer.uploads.first+ expect(upload[:upload_id]).to eq("abc123XYZ")+ expect(result).to eq("![#{upload[:placeholder]}](https://elsewhere.example.com/x.png)")+ end++ it "defers both images of an old lightbox (a thumbnail linking to the full image)" do+ thumb = "/uploads/default/optimized/2X/a/ab/#{sha1}_2_100x75.png"+ full = "/uploads/default/original/2X/a/ab/#{sha1}.png"+ result = extract("[](#{full})")++ expect(buffer.uploads.size).to eq(2)+ expect(buffer.uploads.map { |u| u[:original_markdown] }).to eq(["", full])+ thumb_ph, full_ph = buffer.uploads.map { |u| u[:placeholder] }+ expect(result).to eq("[#{thumb_ph}](#{full_ph})")+ end+ end++ describe "upload paths outside the URL path" do+ # The storage shape counts only inside the URL's own path. A redirect+ # whose query or fragment carries an upload path is the author's link to+ # the redirect, not to the file — rewriting it as an upload would swap+ # the link for the file it points at.+ let(:upload_path) { "/uploads/default/original/1X/#{sha1}.png" }++ it "leaves a bare redirect URL with an upload path in its query alone" do+ raw = "go to https://forum.example.com/redirect?to=#{upload_path} now"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ expect(extractor.engine_refusals).to be_empty+ end++ it "leaves a markdown link with an upload path in its query alone" do+ raw = "[file](https://cdn.example.com/download?file=#{upload_path})"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ expect(extractor.engine_refusals).to be_empty+ end++ it "leaves an image with an upload path in its query alone" do+ raw = ""++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ expect(extractor.engine_refusals).to be_empty+ end++ it "leaves a URL with an upload path in its fragment alone" do+ raw = "see https://forum.example.com/page##{upload_path} here"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ expect(extractor.engine_refusals).to be_empty+ end+ end++ describe "upload URLs with a query" do+ # A signed CDN URL must be taken whole or not at all: replacing only a+ # prefix would leave the signature dangling after the placeholder.+ it "takes a bare upload URL with a long signed query whole" do+ query =+ "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=#{"a" * 320}&X-Amz-Signature=#{"b" * 64}"+ url = "https://cdn.example.com/uploads/default/original/1X/#{sha1}.png?#{query}"+ result = extract("file #{url} end")++ upload = buffer.uploads.first+ expect(upload).to include(upload_id: sha1, original_markdown: url)+ expect(result).to eq("file #{upload[:placeholder]} end")+ end++ it "takes an upload URL with a query whole in a markdown link" do+ url = "https://cdn.example.com/uploads/default/original/1X/#{sha1}.png?v=2"+ result = extract("[file](#{url})")++ upload = buffer.uploads.first+ expect(upload).to include(upload_id: sha1, original_markdown: "[file](#{url})")+ expect(result).to eq(upload[:placeholder])+ end++ it "leaves a URL whose query exceeds the cap alone instead of matching a prefix" do+ url = "https://cdn.example.com/uploads/default/original/1X/#{sha1}.png?x=#{"a" * 1200}"+ raw = "file #{url} end"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ expect(extractor.engine_refusals).to be_empty+ end++ # Same rule for a wordless over-cap query: 2,000 percent signs are not a+ # short linkify tail, and a prefix match would cut the URL mid-query.+ it "leaves a bare URL with an over-cap punctuation-only query alone" do+ url = "https://cdn.example.com/uploads/default/original/1X/#{sha1}.png?x=#{"%" * 2000}"+ raw = "file #{url} end"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ expect(extractor.engine_refusals).to be_empty+ end++ it "leaves a markdown link with an over-cap punctuation-only query alone" do+ url = "https://cdn.example.com/uploads/default/original/1X/#{sha1}.png?x=#{"%" * 2000}"+ raw = "[file](#{url})"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ expect(extractor.engine_refusals).to be_empty+ end++ it "takes a short URL with a query whole in a markdown link" do+ result = extract("[file](/uploads/short-url/21.png?v=2)")++ upload = buffer.uploads.first+ expect(upload).to include(original_markdown: "[file](/uploads/short-url/21.png?v=2)")+ expect(result).to eq(upload[:placeholder])+ end++ # A destination may end in a bare `?` or `#`; the engine keeps the+ # marker in the href, so the construct takes it as part of the URL.+ it "takes a short URL ending in a bare query marker whole" do+ result = extract("[file](/uploads/short-url/21.png?)")++ upload = buffer.uploads.first+ expect(upload).to include(original_markdown: "[file](/uploads/short-url/21.png?)")+ expect(result).to eq(upload[:placeholder])+ expect(extractor.engine_refusals).to be_empty+ end++ it "takes a full upload URL ending in a bare query marker whole" do+ url = "https://cdn.example.com/uploads/default/original/1X/#{sha1}.png?"+ result = extract("[file](#{url})")++ upload = buffer.uploads.first+ expect(upload).to include(upload_id: sha1, original_markdown: "[file](#{url})")+ expect(result).to eq(upload[:placeholder])+ expect(extractor.engine_refusals).to be_empty+ end++ it "takes a full upload URL ending in a bare fragment marker whole" do+ url = "https://cdn.example.com/uploads/default/original/1X/#{sha1}.png#"+ result = extract("[file](#{url})")++ upload = buffer.uploads.first+ expect(upload).to include(upload_id: sha1, original_markdown: "[file](#{url})")+ expect(result).to eq(upload[:placeholder])+ expect(extractor.engine_refusals).to be_empty+ end+ end++ describe "S3/CDN storage URLs without an uploads segment" do+ # Core's S3 store writes the `original|optimized/` path directly under+ # the bucket host or a bucket prefix, with no `/uploads/` segment.+ it "defers a bare absolute storage URL" do+ url = "https://cdn.example.com/original/1X/#{sha1}.png"+ result = extract("pic #{url} here")++ upload = buffer.uploads.first+ expect(upload).to include(upload_id: sha1, original_markdown: url)+ expect(result).to eq("pic #{upload[:placeholder]} here")+ end++ it "defers a protocol-relative bucket-prefixed storage URL in an image" do+ url = "//cdn.example.com/bucket/optimized/2X/a/#{sha1}_2_690x388.png"+ result = extract("")++ upload = buffer.uploads.first+ expect(upload).to include(upload_id: sha1, original_markdown: "")+ expect(result).to eq(upload[:placeholder])+ end++ it "leaves a relative storage path without an uploads segment alone" do+ # No local store writes a relative `/original/…` path; recognizing one+ # would rewrite unrelated site paths that happen to share the name.+ raw = "[x](/original/1X/#{sha1}.png)"++ expect(extract(raw)).to eq(raw)+ expect(buffer.uploads).to be_empty+ expect(extractor.engine_refusals).to be_empty+ end+ end++ # A reference-style image `![alt][1]` keeps its alt text and dimensions where+ # they are and takes its source from the definition line, so the definition's+ # destination is the whole recorded span.+ describe "uploads behind a reference definition" do+ let(:internal_link_hosts) { { source_host => nil } }++ it "defers the destination of a short-URL definition" do+ url = "upload://wfnzm0tBLXg6BRQWnIoNfnl8HNs.jpg"+ result = extract("![a|1x1][1]\n\n[1]: #{url}")++ upload = buffer.uploads.first+ expect(upload).to include(upload_id: "wfnzm0tBLXg6BRQWnIoNfnl8HNs", original_markdown: url)+ expect(result).to eq("![a|1x1][1]\n\n[1]: #{upload[:placeholder]}")+ expect(extractor.engine_refusals).to be_empty+ end++ it "defers an angle-bracket short-URL definition" do+ url = "upload://wfnzm0tBLXg6BRQWnIoNfnl8HNs.jpg"+ result = extract("![a|1x1][1]\n\n[1]: <#{url}>")++ upload = buffer.uploads.first+ expect(upload).to include(upload_id: "wfnzm0tBLXg6BRQWnIoNfnl8HNs", original_markdown: url)+ expect(result).to eq("![a|1x1][1]\n\n[1]: <#{upload[:placeholder]}>")+ expect(extractor.engine_refusals).to be_empty+ end++ it "defers short-URL definition continuations while preserving code copies" do+ url = "upload://wfnzm0tBLXg6BRQWnIoNfnl8HNs.jpg"++ ["\n", "\r\n", "\r"].each do |newline|+ [url, "<#{url}>"].each do |destination|+ buffer.clear+ prefix = "![a|1x1][1]#{newline}#{newline}[1]:#{newline} "+ suffix = "#{newline}#{newline}`#{url}`"++ result = extract("#{prefix}#{destination}#{suffix}")++ expect(buffer.uploads).to contain_exactly(+ include(upload_id: "wfnzm0tBLXg6BRQWnIoNfnl8HNs", original_markdown: url),+ )+ placeholder = buffer.uploads.first[:placeholder]+ expect(result).to eq("#{prefix}#{destination.sub(url, placeholder)}#{suffix}")+ expect(extractor.engine_refusals).to be_empty+ end+ end+ end++ it "defers the destination of a full-URL definition" do+ url = "https://forum.example.com/uploads/default/original/1X/#{sha1}.png"+ result = extract("![a][1]\n\n[1]: #{url}")++ upload = buffer.uploads.first+ expect(upload).to include(upload_id: sha1, original_markdown: url, external_host: nil)+ expect(result).to eq("![a][1]\n\n[1]: #{upload[:placeholder]}")+ expect(extractor.engine_refusals).to be_empty+ end++ it "marks a foreign host on a definition destination" do+ url = "https://cdn.example.org/uploads/default/original/1X/#{sha1}.png"+ extract("![a][1]\n\n[1]: #{url}")++ expect(buffer.uploads.first).to include(upload_id: sha1, external_host: "cdn.example.org")+ end++ it "keeps a definition-shaped line inside a fence verbatim" do+ url = "upload://wfnzm0tBLXg6BRQWnIoNfnl8HNs.jpg"+ raw = "![a|1x1][1]\n\n[1]: #{url}\n\n```\n[2]: #{url}\n```\n"+ result = extract(raw)++ expect(buffer.uploads.size).to eq(1)+ expect(result).to include("```\n[2]: #{url}\n```")+ expect(result).to include("[1]: #{buffer.uploads.first[:placeholder]}")+ end+ end++ describe "multi-line alt text" do+ it "defers an image whose alt text runs to several thousand bytes" do+ alt = "The image shows a chat window. " * 100+ raw = ""++ result = extract(raw)++ expect(buffer.uploads.size).to eq(1)+ expect(result).to eq(buffer.uploads.first[:placeholder])+ end++ it "defers an image whose alt text spans lines" do+ raw =+ " tail"++ result = extract(raw)++ expect(buffer.uploads.size).to eq(1)+ upload = buffer.uploads.first+ expect(upload[:upload_id]).to eq("k9k7D6IBBk6Sg7EuRM59QL72xXn")+ expect(upload[:original_markdown]).to eq(raw.delete_suffix(" tail"))+ expect(result).to eq("#{upload[:placeholder]} tail")+ end+ end++ describe "lightbox links" do+ it "defers the destination of a link wrapping its own image" do+ # Core renders an image inside an upload link as the alt text, so the+ # inner image is no construct; the destination is the one live upload.+ short = "upload://bnqOrMUuNaz4vwpObqnsFgpgPUF.jpeg"+ result = extract("[](#{short})")++ expect(extractor.engine_refusals).to be_empty+ expect(buffer.uploads.size).to eq(1)+ expect(buffer.uploads.first).to include(+ upload_id: "bnqOrMUuNaz4vwpObqnsFgpgPUF",+ original_markdown: short,+ )+ expect(result).to eq("[](#{buffer.uploads.first[:placeholder]})")+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb b/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rbindex ec53ced320a..cb1654f8d9d 100644--- a/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb+++ b/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb@@ -26,6 +26,7 @@ RSpec.describe Migrations::Converters::EmbedBuffer doquoted_user_id: 2,quoted_username: "bob",quoted_name: "Bob B",+ original_markdown: "[quote=\"bob, post:3, topic:8\"]",)expect(buffer.quotes).to contain_exactly(@@ -37,6 +38,7 @@ RSpec.describe Migrations::Converters::EmbedBuffer doquoted_user_id: 2,quoted_username: "bob",quoted_name: "Bob B",+ original_markdown: "[quote=\"bob, post:3, topic:8\"]",},)end@@ -48,6 +50,8 @@ RSpec.describe Migrations::Converters::EmbedBuffer dotext: "here",target_type: Migrations::Database::IntermediateDB::Enums::LinkTarget::TOPIC,target_id: 9,+ original_markdown: "[here](https://example.com)",+ url_offset: 7,)expect(buffer.links).to contain_exactly(@@ -60,16 +64,32 @@ RSpec.describe Migrations::Converters::EmbedBuffer dotarget_name: nil,target_topic_id: nil,target_post_number: nil,+ target_tag_path: nil,target_suffix: nil,+ original_markdown: "[here](https://example.com)",+ url_offset: 7,+ label_url_offset: nil,},)endit "records a mention descriptor keyed for IntermediateDB::EmbedMention" do- token = buffer.mention(mention_type: mention_type::USER, target_id: 7, name: "bob")+ token =+ buffer.mention(+ mention_type: mention_type::USER,+ target_id: 7,+ name: "bob",+ original_markdown: "@bob",+ )expect(buffer.mentions).to contain_exactly(- { placeholder: token, mention_type: mention_type::USER, target_id: 7, name: "bob" },+ {+ placeholder: token,+ mention_type: mention_type::USER,+ target_id: 7,+ name: "bob",+ original_markdown: "@bob",+ },)end@@ -87,7 +107,13 @@ RSpec.describe Migrations::Converters::EmbedBuffer doendit "records a hashtag descriptor keyed for IntermediateDB::EmbedHashtag" do- token = buffer.hashtag(hashtag_type: hashtag_type::CATEGORY, target_id: nil, name: "support")+ token =+ buffer.hashtag(+ hashtag_type: hashtag_type::CATEGORY,+ target_id: nil,+ name: "support",+ original_markdown: "#support",+ )expect(buffer.hashtags).to contain_exactly({@@ -95,6 +121,7 @@ RSpec.describe Migrations::Converters::EmbedBuffer dohashtag_type: hashtag_type::CATEGORY,target_id: nil,name: "support",+ original_markdown: "#support",},)end@@ -134,7 +161,12 @@ RSpec.describe Migrations::Converters::EmbedBuffer dotoken = buffer.upload(upload_id: "abc123", original_markdown: "")expect(buffer.uploads).to contain_exactly(- { placeholder: token, upload_id: "abc123", original_markdown: "" },+ {+ placeholder: token,+ upload_id: "abc123",+ original_markdown: "",+ external_host: nil,+ },)enddiff --git a/migrations/converters/spec/lib/migrations/converters/markdown_engine/bundle_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_engine/bundle_spec.rbnew file mode 100644index 00000000000..c39278b9e99--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/markdown_engine/bundle_spec.rb@@ -0,0 +1,134 @@+# frozen_string_literal: true++require "open3"+require "tmpdir"++RSpec.describe Migrations::Converters::MarkdownEngine::Bundle do+ subject(:bundle) { described_class.load_or_build }++ let(:names) { bundle.entries.map(&:first) }++ it "loads the host's precompiled pretty-text bundle first" do+ expect(names.first).to eq("pretty-text.js")+ expect(bundle.entries.first.last).to include("__PrettyText")+ end++ it "contains the bundled plugins' vendored libraries" do+ expect(names).to include(a_string_matching(%r{node_modules/moment/moment\.js\z}))+ end++ it "contains the bundled plugins' markdown features" do+ expect(names).to include(+ a_string_matching(%r{\Adiscourse/plugins/poll/.*discourse-markdown/poll\z}),+ )+ end++ it "ends with the emoji table" do+ expect(names.last).to eq("migrations/emoji-data")+ end++ it "caches the built bundle on disk and reuses it" do+ bundle+ cache_dir =+ File.join(Migrations::Converters::MarkdownEngine.discourse_root, described_class::CACHE_DIR)+ cache_files = Dir[File.join(cache_dir, "markdown-engine-bundle-*.json")]+ expect(cache_files.size).to eq(1)++ # A rebuild from the warm cache must not transpile.+ allow(AssetProcessor).to receive(:new).and_call_original+ expect(described_class.load_or_build.entries.size).to eq(bundle.entries.size)+ expect(AssetProcessor).not_to have_received(:new)+ end++ it "builds a cold cache in a subprocess, leaving this process V8- and shim-free" do+ Dir.mktmpdir do |dir|+ cold = described_class.load_or_build(cache_dir: dir)++ expect(cold.entries.size).to eq(bundle.entries.size)+ # The whole point of the subprocess: the forking converter parent must+ # never initialize V8 (multithreaded V8 is not fork-safe) nor see the+ # Rails/Discourse stand-ins the transpiler needs.+ expect(AssetProcessor.booted?).to be(false)+ expect(defined?(::Rails)).to be_nil+ expect(defined?(::Discourse)).to be_nil++ cache_files = Dir[File.join(dir, "markdown-engine-bundle-*.json")]+ expect(cache_files.size).to eq(1)++ # A truncated cache (for example a partial copy) is a miss that+ # rebuilds, never a crash or trusted junk.+ File.write(cache_files.first, "{\"entries\": [[\"x\",")+ rebuilt = described_class.load_or_build(cache_dir: dir)+ expect(rebuilt.entries.size).to eq(bundle.entries.size)++ # Valid JSON with the wrong shape is a miss too; trusting it would+ # raise a NoMethodError later, in the context.+ File.write(cache_files.first, "{\"entries\": \"corrupt\"}")+ reshaped = described_class.load_or_build(cache_dir: dir)+ expect(reshaped.entries.size).to eq(bundle.entries.size)++ # An empty entry list can only come from a damaged file — no build+ # produces zero entries — so it is a miss, not a bundle with no engine.+ File.write(cache_files.first, "{\"entries\": []}")+ refilled = described_class.load_or_build(cache_dir: dir)+ expect(refilled.entries.size).to eq(bundle.entries.size)+ end+ end++ it "supports every host plugin that ships markdown rules" do+ root = Migrations::Converters::MarkdownEngine.discourse_root+ shipping =+ Dir[+ File.join(root, "plugins", "*", "assets/javascripts/**/discourse-markdown/**/*.{js,js.es6}")+ ].map { |path| path.delete_prefix("#{File.join(root, "plugins")}/").split("/").first }.uniq++ # discourse-ai's markdown modules only allowlist HTML attributes and+ # register no rules, so nothing tokenizes differently without them. Any+ # other unlisted plugin would have its syntax scanned as ordinary prose —+ # candidate text inside it would be replaced — so a new plugin must be+ # added to the supported list (with its settings in Config::SETTING_KEYS)+ # or excluded here with a reason.+ expect(shipping - %w[discourse-ai]).to match_array(described_class::CORE_MARKDOWN_PLUGINS)+ end++ it "supports the production fork model: bundle in the parent, context in the worker" do+ # Run in a fresh ruby process so this spec process's own V8 contexts (other+ # examples build them) can't mask or break the property under test: a+ # parent that only loaded the bundle forks, and the worker builds a+ # working context.+ program = <<~RUBY+ require "migrations-converters"++ bundle = Migrations::Converters::MarkdownEngine::Bundle.load_or_build+ raise "parent booted V8" if AssetProcessor.booted?++ reader, writer = IO.pipe+ pid =+ fork do+ reader.close+ config = Migrations::Converters::MarkdownEngine::Config.new+ context = Migrations::Converters::MarkdownEngine::Context.new(bundle:, config:)+ result = context.scan([{ id: 1, raw: "hello @sam" }]).first+ writer.write(result["blocks"].first["mentions"].join(","))+ writer.close+ exit!(0)+ end+ writer.close+ output = reader.read+ Process.wait(pid)+ raise "worker scan failed: \#{output.inspect}" unless output == "@sam"+ puts "worker-scan-ok"+ RUBY++ stdout, stderr, status =+ Open3.capture3(+ RbConfig.ruby,+ "-e",+ program,+ chdir: Migrations::Converters::MarkdownEngine.discourse_root,+ )++ expect(status.success?).to be(true), stderr+ expect(stdout).to include("worker-scan-ok")+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/markdown_engine/config_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_engine/config_spec.rbnew file mode 100644index 00000000000..2f4728129cd--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/markdown_engine/config_spec.rb@@ -0,0 +1,224 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::MarkdownEngine::Config do+ subject(:config) { described_class.new }++ describe "default settings" do+ it "reads the checkout's YAML defaults" do+ expect(config.settings["enable_mentions"]).to be(true)+ expect(config.settings["unicode_usernames"]).to be(false)+ expect(config.settings["poll_maximum_options"]).to be_an(Integer)+ expect(config.settings["markdown_linkify_tlds"]).to be_a(String)+ end++ it "covers every setting key" do+ expect(config.settings.keys).to match_array(described_class::SETTING_KEYS)+ end+ end++ describe "source overrides" do+ subject(:config) do+ described_class.new(+ source_settings: {+ "enable_mentions" => "f",+ "poll_maximum_options" => "5",+ "emoji_set" => "noto",+ "not_a_markdown_setting" => "ignored",+ "max_username_length" => 42,+ },+ )+ end++ it "coerces stringly-typed database values against the default's type" do+ expect(config.settings["enable_mentions"]).to be(false)+ expect(config.settings["poll_maximum_options"]).to eq(5)+ expect(config.settings["emoji_set"]).to eq("noto")+ end++ it "ignores settings the markdown pipeline does not read" do+ expect(config.settings).not_to have_key("not_a_markdown_setting")+ expect(config.settings).not_to have_key("max_username_length")+ end+ end++ describe "name sets" do+ subject(:config) do+ described_class.new(+ category_slugs: %w[Support FEATURE],+ tag_names: %w[Bug],+ custom_emoji_names: %w[partyparrot],+ )+ end++ it "normalizes slugs and tag names the way the constructs do" do+ expect(config.category_slugs).to eq(%w[support feature])+ expect(config.tag_names).to eq(%w[bug])+ end++ it "applies NFC, so a decomposed configured name matches its composed form" do+ config = described_class.new(tag_names: ["café"])+ expect(config.tag_names).to eq([Migrations::NameNormalizer.normalize("café")])+ end++ it "derives the engine's lookup set from the leaf slug of a category path" do+ config = described_class.new(category_slugs: %w[Support:Billing news])+ # The engine resolves `#support:billing` by its child slug, the way the+ # destination's lookup does.+ expect(config.category_lookup_slugs).to eq(%w[billing news])+ end++ it "drops a dangling separator and duplicate leaves from the lookup set" do+ config = described_class.new(category_slugs: ["support:", "a:news", "b:news", ""])+ expect(config.category_lookup_slugs).to eq(%w[support news])+ end++ it "exposes the extractor's name set as full paths plus tag names" do+ config = described_class.new(category_slugs: %w[support:billing], tag_names: %w[Bug])+ names = config.hashtag_names++ expect(names).to be_a(Migrations::CompactStringSet)+ expect(names).to include("support:billing")+ expect(names).to include("bug")+ # The extractor resolves what a post spells, so the leaf alone is not a+ # name the source has.+ expect(names).not_to include("billing")+ end++ it "builds a custom emoji map keyed by name" do+ expect(config.custom_emoji).to eq("partyparrot" => "/images/emoji/custom/partyparrot.png")+ end+ end++ describe "avatar sizes" do+ it "parses the pipe-separated YAML default into integers" do+ expect(config.avatar_sizes).to all(be_an(Integer))+ expect(config.avatar_sizes).not_to be_empty+ end+ end++ # The engine only receives the settings named in SETTING_KEYS, so any+ # `siteSettings` read added to the JavaScript the Bundle loads must be+ # classified here before it silently falls back to `undefined` in V8.+ describe "SETTING_KEYS coverage" do+ # Recognized read idioms, each capturing the key(s) read. Anything that+ # mentions `siteSettings` and matches none of these fails the second+ # example: an alias or a new idiom must be added here, not silently skipped.+ let(:property_read) { /siteSettings\??\.([A-Za-z_0-9]+)/ }+ let(:bracket_read) { /siteSettings\??\[\s*["']([A-Za-z_0-9]+)["']\s*\]/ }+ let(:destructuring_read) { /\{([^{}]*)\}\s*=[^=;\n]*siteSettings/ }+ # A callback parameter named `siteSettings` declares the object, it reads+ # no key — the reads happen through the other idioms in the body.+ let(:parameter_declaration) { /[(,]\s*siteSettings\s*(?=[,)])/ }++ def bundled_js_files+ root = Migrations::Converters::MarkdownEngine.discourse_root+ bundle = Migrations::Converters::MarkdownEngine::Bundle++ files = []+ # Only the JavaScript sources — the core-bundle globs also cover build+ # configs and the lockfile, which read no site settings.+ bundle.core_bundle_globs.each do |pattern|+ files.concat(Dir.glob(pattern, base: root).grep(/\.js\z/).map { |f| File.join(root, f) })+ end+ bundle::CORE_MARKDOWN_PLUGINS.each do |plugin|+ files.concat(+ Dir[+ File.join(+ root,+ "plugins",+ plugin,+ "assets/javascripts/**/discourse-markdown/**/*.{js,js.es6}",+ )+ ],+ )+ end+ files+ end++ def keys_read_in(files)+ files.flat_map do |file|+ source = File.read(file)+ source.scan(property_read).flatten + source.scan(bracket_read).flatten ++ source+ .scan(destructuring_read)+ .flatten+ .flat_map { |names| names.split(",").map { |name| name[/[A-Za-z_0-9]+/] } }+ .compact+ end+ end++ # `avatar_sizes` is covered by the dedicated AVATAR_SIZES_KEY: it rides+ # the options as `avatar_sizes`, and the processor's helpers shim hands+ # it back to avatar-utils under a `siteSettings` reading.+ def covered_keys+ described_class::SETTING_KEYS + [described_class::AVATAR_SIZES_KEY]+ end++ it "contains every siteSettings key the bundled markdown JavaScript reads" do+ expect(keys_read_in(bundled_js_files).uniq - covered_keys).to be_empty+ end++ # The supported plugin list is an allowlist because of this: each listed+ # plugin's own reads are covered, so its rules run with the source+ # site's settings. A plugin added to the list without extending+ # SETTING_KEYS fails here by name.+ it "covers each supported plugin's own siteSettings reads" do+ root = Migrations::Converters::MarkdownEngine.discourse_root++ Migrations::Converters::MarkdownEngine::Bundle::CORE_MARKDOWN_PLUGINS.each do |plugin|+ files =+ Dir[+ File.join(+ root,+ "plugins",+ plugin,+ "assets/javascripts/**/discourse-markdown/**/*.{js,js.es6}",+ )+ ]+ # A supported plugin with no markdown files would make this example+ # pass with nothing checked — that happens when the plugin's file+ # layout changes, and it means the plugin is no longer bundled.+ expect(files).not_to be_empty,+ "#{plugin} has no markdown feature files under the known path"+ missing = keys_read_in(files).uniq - covered_keys+ expect(missing).to be_empty,+ "#{plugin} reads siteSettings keys SETTING_KEYS does not cover: #{missing.inspect}"+ end+ end++ it "recognizes every siteSettings mention, so aliases can't hide a read" do+ unrecognized =+ bundled_js_files.flat_map do |file|+ File+ .read(file)+ .each_line+ .with_index(1)+ .filter_map do |line, number|+ line = line.sub(%r{//.*}, "")+ next unless line.include?("siteSettings")+ # Object shorthand on its own line: a multiline destructuring or+ # parameter list declaring the object, same as the inline forms.+ next if line.match?(/\A\s*siteSettings,?\s*\z/)++ # An object-literal key (`{ siteSettings: {} }`) supplies the+ # object; it reads no key.+ stripped =+ line+ .gsub(property_read, "")+ .gsub(bracket_read, "")+ .gsub(destructuring_read, "")+ .gsub(parameter_declaration, "(")+ .gsub(/siteSettings\s*:/, "")+ # What remains is a mention none of the read idioms account+ # for: a bare reference (an alias assignment, a helper call+ # forwarding the object) whose later reads this static check+ # cannot see.+ "#{file}:#{number}: #{line.strip}" if stripped.include?("siteSettings")+ end+ end++ expect(unrecognized).to be_empty,+ -> { "extend the recognized siteSettings read idioms:\n#{unrecognized.join("\n")}" }+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/markdown_engine/context_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_engine/context_spec.rbnew file mode 100644index 00000000000..f1ada2a0c15--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/markdown_engine/context_spec.rb@@ -0,0 +1,196 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::MarkdownEngine::Context do+ subject(:context) { described_class.new(bundle:, config:) }++ let(:bundle) { Migrations::Converters::MarkdownEngine::Bundle.load_or_build }+ let(:config) do+ Migrations::Converters::MarkdownEngine::Config.new(+ category_slugs: %w[support],+ tag_names: %w[bug],+ custom_emoji_names: %w[partyparrot],+ )+ end++ after { context.close }++ def scan_one(raw)+ context.scan([{ id: 1, raw: }]).first+ end++ it "rebuilds lazily after a reset" do+ expect(scan_one("hi @sam").dig("blocks", 0, "mentions")).to eq(["@sam"])++ context.reset!+ # The engine failure that triggered the reset cost its body; the next+ # body gets a fresh context transparently.+ expect(scan_one("hi @sam").dig("blocks", 0, "mentions")).to eq(["@sam"])+ end++ it "widens the isolate's ceiling when a scan asks for more time" do+ expect(scan_one("hi @sam").dig("blocks", 0, "mentions")).to eq(["@sam"])++ result = context.scan([{ id: 1, raw: "hi @sam" }], timeout_ms: 30_000).first+ expect(result.dig("blocks", 0, "mentions")).to eq(["@sam"])+ end++ # A caller counting a per-body deadline down lowers the ceiling on every+ # call, so a lower ceiling must not cost an isolate: MiniRacer fixes its+ # timeout at construction, and rebuilding evaluates the whole bundle again.+ it "keeps one isolate while the ceiling counts down" do+ context.scan([{ id: 1, raw: "warm up" }], timeout_ms: 5_000)++ allow(MiniRacer::Context).to receive(:new).and_call_original++ [2_000, 1_000, 500].each do |timeout_ms|+ result = context.scan([{ id: 1, raw: "hi @sam" }], timeout_ms:).first+ expect(result.dig("blocks", 0, "mentions")).to eq(["@sam"])+ end++ expect(MiniRacer::Context).not_to have_received(:new)+ end++ it "terminates a body that outruns a ceiling below the isolate's" do+ context.scan([{ id: 1, raw: "warm up" }], timeout_ms: 5_000)++ # No post is guaranteed to parse forever, so the scan entry point is+ # replaced with one that does — the isolate's own ceiling is five seconds+ # here, and only the enforced 200 ms can end this call sooner.+ context.instance_variable_get(:@context).eval("__scanPosts = () => { while (true) {} };")++ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)+ expect { context.scan([{ id: 1, raw: "x" }], timeout_ms: 200) }.to raise_error(+ MiniRacer::ScriptTerminatedError,+ )+ expect(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).to be < 2+ end++ # Empty-destination links once looped the label-hit counter forever+ # (indexOf with an empty needle never advances), burning the full parse+ # timeout on bodies as small as four bytes.+ it "scans empty-destination and empty-label links in ordinary time" do+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)+ ["[]()", "[a]()", "[](https://)", "[Uploading: t.jpg...]()", "sure i will ;[]())"].each do |raw|+ expect { scan_one(raw) }.not_to raise_error+ end+ expect(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).to be < 2+ end++ it "resolves hashtags across Unicode composition forms, but not compat forms" do+ composed = "café"+ decomposed = "café"+ unicode_context =+ described_class.new(+ bundle:,+ config: Migrations::Converters::MarkdownEngine::Config.new(tag_names: [composed, "fix"]),+ )++ begin+ # NFC + downcase on both sides (Ruby's NameNormalizer, the JS lookup):+ # either composition form in a post denotes the same configured name.+ [composed, decomposed, "CafÉ"].each do |spelling|+ result = unicode_context.scan([{ id: 1, raw: "tagged ##{spelling}" }]).first+ expect(result.dig("blocks", 0, "hashtags", 0, "type")).to eq("tag"),+ "##{spelling.inspect} did not resolve"+ end++ # NFC deliberately keeps compatibility characters distinct (NameNormalizer+ # is NFC, not NFKC): the `fi` ligature does not denote the `fix` tag.+ result = unicode_context.scan([{ id: 1, raw: "tagged #fix" }]).first+ expect(result.dig("blocks", 0, "hashtags").to_a).to be_empty+ ensure+ unicode_context.close+ end+ end++ it "extracts mentions with the block's line map" do+ result = scan_one("a paragraph\n\nwith @sam here")+ expect(result["blocks"]).to contain_exactly(+ a_hash_including("mentions" => ["@sam"], "map" => [2, 3]),+ )+ end++ it "shields code spans and code blocks the way core does" do+ result = scan_one("`@code` and @prose\n\n```\n@fence\n```")+ expect(result["blocks"].map { |block| block["mentions"] }).to eq([["@prose"]])+ expect(result["blocks"].first["code"]).to eq(1)+ expect(result["blockTokens"]).to include(a_hash_including("type" => "fence", "map" => [2, 5]))+ end++ it "resolves hashtags against the source name sets only" do+ result = scan_one("#support #bug #unknown")+ expect(result["blocks"].first["hashtags"]).to contain_exactly(+ { "type" => "category", "slug" => "support" },+ { "type" => "tag", "slug" => "bug" },+ )+ end++ it "keeps unresolved short upload URLs as the construct value" do+ result = scan_one("")+ expect(result["blocks"].first["images"]).to eq(%w[upload://2Yjf3WE4KOQ88YUb4fUMubKB9My.png])+ end++ it "collects linkified bare domains and explicit links" do+ result = scan_one("see meta.example.com/t/topic/123 and [x](https://example.com/page)")+ expect(result["blocks"].first["links"]).to eq(+ [+ # A linkified bare domain is its own label but exists once in the raw,+ # so it contributes no label occurrences.+ { "href" => "http://meta.example.com/t/topic/123", "labelHits" => 0 },+ { "href" => "https://example.com/page", "labelHits" => 0 },+ ],+ )+ end++ it "reports a reference definition's destination, not the uses that resolve through it" do+ result = scan_one("see [a][1] and ![b][1]\n\n[1]: https://example.com/ref\n")+ expect(result["blocks"]).to contain_exactly(+ a_hash_including(+ "map" => [2, 3],+ "links" => [{ "href" => "https://example.com/ref", "labelHits" => 0 }],+ ),+ )+ end++ it "reports only the first definition of a repeated label" do+ result = scan_one("[x][1]\n\n[1]: https://example.com/one\n[1]: https://example.com/two\n")+ expect(result["blocks"].flat_map { |block| block["links"] }).to eq(+ [{ "href" => "https://example.com/one", "labelHits" => 0 }],+ )+ end++ it "counts a self-link's destination appearing in its label" do+ result = scan_one("[https://example.com/page](https://example.com/page)")+ expect(result["blocks"].first["links"]).to eq(+ [{ "href" => "https://example.com/page", "labelHits" => 1 }],+ )+ end++ it "records quote extents from the mapped bbcode token" do+ result = scan_one(%{[quote="sam, post:1, topic:2"]\nquoted @sam\n[/quote]})+ expect(result["blockTokens"]).to include(+ a_hash_including("type" => "bbcode_open", "tag" => "blockquote", "map" => [0, 2]),+ )+ expect(result["blocks"].first).to include("mentions" => ["@sam"], "map" => [1, 2])+ end++ it "recognizes standard, unicode, and source custom emoji" do+ result = scan_one(":smile: :partyparrot: 😀 :not_an_emoji:")+ expect(result["blocks"].first["emojis"]).to eq(%w[smile partyparrot grinning_face])+ end++ it "scrubs invalid encoding instead of raising" do+ raw = "hi @sam \xC3".b+ expect(scan_one(raw)["blocks"].first["mentions"]).to eq(["@sam"])+ end++ it "returns post ids with their results" do+ results = context.scan([{ id: 7, raw: "@sam" }, { id: 9, raw: "plain" }])+ expect(results.map { |result| result["id"] }).to eq([7, 9])+ end++ it "refuses to scan after being discarded" do+ context.discard!+ expect { context.scan([{ id: 1, raw: "x" }]) }.to raise_error(described_class::DiscardedError)+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/markdown_engine/emoji_data_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_engine/emoji_data_spec.rbnew file mode 100644index 00000000000..c29676ed452--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/markdown_engine/emoji_data_spec.rb@@ -0,0 +1,30 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::MarkdownEngine::EmojiData do+ describe ".unicode_replacements" do+ subject(:replacements) { described_class.unicode_replacements }++ it "maps unicode emoji to their names" do+ expect(replacements["😀"]).to eq("grinning_face")+ end++ it "includes skin-toned variants for tonable emoji" do+ expect(replacements.values).to include(a_string_matching(/:t2\z/))+ end++ it "keeps the symbol-like characters unmapped" do+ expect(replacements.values).not_to include("registered", "copyright", "trade_mark")+ end++ it "applies the extra text-symbol replacements" do+ expect(replacements["\u{2665}"]).to eq("heart")+ end+ end++ describe ".set_unicode_source" do+ it "produces a setUnicode call" do+ expect(described_class.set_unicode_source).to start_with("__PrettyText.setUnicode({")+ expect(described_class.set_unicode_source).to end_with("});")+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/markdown_engine/parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_engine/parity_spec.rbnew file mode 100644index 00000000000..1776491221b--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/markdown_engine/parity_spec.rb@@ -0,0 +1,173 @@+# frozen_string_literal: true++# The acceptance test for the self-contained engine context: the same posts+# scanned through (a) the standalone `Context` and (b) PrettyText's own booted+# engine must produce identical output. Any wrongly stubbed callback or+# mis-mapped option surfaces as a diff here. Needs a booted Rails environment,+# so it is tagged `:rails` and runs only under `MIGRATIONS_RAILS=1`.+RSpec.describe Migrations::Converters::MarkdownEngine::Context, :rails do+ let!(:category) { Fabricate(:category, slug: "parity-cat") }+ let!(:tag) { Fabricate(:tag, name: "paritytag") }++ let(:config) do+ source_settings =+ Migrations::Converters::MarkdownEngine::Config::SETTING_KEYS+ .filter_map { |key| [key, SiteSetting.public_send(key)] if SiteSetting.respond_to?(key) }+ .to_h++ Migrations::Converters::MarkdownEngine::Config.new(+ source_settings:,+ category_slugs: [category.slug],+ tag_names: [tag.name],+ additional_options: Site.markdown_additional_options,+ )+ end++ let(:context) do+ described_class.new(+ bundle: Migrations::Converters::MarkdownEngine::Bundle.load_or_build,+ config:,+ )+ end++ after { context.close }++ # Core constructs only: whether plugin features (poll etc.) are present on+ # the PrettyText side depends on which plugins the Rails process loaded, so+ # plugin bbcode has no stable parity target here.+ let(:fixtures) do+ [+ "plain paragraph, nothing to find",+ "a mention @someuser and a trailing-dot one @someuser.",+ "`@code` but @someuser\n\n indented @code\n",+ "```\nfenced @code\n```\n\n@someuser",+ "#parity-cat and #paritytag and #unknown-slug",+ "#parity-cat::category #paritytag::tag",+ ":smile: and 😀 and :not_an_emoji:",+ "",+ "[attachment.pdf|attachment](upload://abcdefghijklmnopqrstuvwxy.pdf)",+ "bare example.com/t/slug/123 and <https://example.com/auto>",+ "[text](https://example.com/page \"a title\") and [ref link][1]\n\n[1]: https://example.com/ref",+ "https://example.com/percent%20encoded and [x](https://example.com/a(b))",+ %{[quote="someuser, post:2, topic:8"]\nquoted @someuser\n[/quote]},+ "> markdown quote with @someuser\n\nheading anchor test\n\n# A Heading",+ "<code>@someuser</code> and <pre>@someuser</pre>",+ "html block:\n\n<div>\n@someuser\n</div>",+ "a\rb\r\nc with @someuser",+ ]+ end++ def pretty_text_scan(posts)+ # `__PrettyText.cook` builds its engine locally, so a hook captures the+ # completed options of a real cook and rebuilds the identical engine as+ # `__pt` for the scan function — the exact configuration the site cooks+ # with, not a re-assembly that could drift.+ PrettyText.v8.eval(<<~JS)+ __parityOriginalCook = __PrettyText.cook;+ __PrettyText.cook = function (text, optInput) {+ const result = __parityOriginalCook.call(this, text, optInput);+ __pt = require("discourse-markdown-it").default+ .withCustomFeatures(require("discourse/static/markdown-it/features").default())+ .withOptions(optInput);+ // The same rule `Context` disables, so both sides see the reference+ // definition tokens the scan counts destinations from.+ __pt.options.engine.core.ruler.disable("strip_references");+ return result;+ };+ JS+ PrettyText.markdown("warm up")+ PrettyText.v8.eval("__PrettyText.cook = __parityOriginalCook;")+ scan_js =+ File.read(+ File.join(+ Migrations::Converters.root_path,+ "lib/migrations/converters/markdown_engine/scan.js",+ ),+ )+ PrettyText.v8.eval(scan_js)+ PrettyText.v8.call("__scanPosts", posts)+ end++ it "produces the same scan output as PrettyText's booted engine" do+ posts = fixtures.each_with_index.map { |raw, index| { "id" => index, "raw" => raw } }++ standalone = context.scan(posts.map { |post| { id: post["id"], raw: post["raw"] } })+ booted = pretty_text_scan(posts)++ standalone.zip(booted).each { |ours, theirs| expect(ours).to eq(theirs), <<~MSG }+ scan mismatch for fixture #{ours["id"]}:+ #{fixtures[ours["id"]].inspect}++ standalone: #{ours.inspect}+ booted: #{theirs.inspect}+ MSG+ end++ # The bundle ships the core-bundled plugins' markdown features because+ # their constructs must tokenize the way the destination cooks them — this+ # asserts exactly that, against each plugin PrettyText actually loaded in+ # this checkout (an absent plugin would make the booted side feature-less+ # and the comparison meaningless, so those fixtures are skipped).+ it "produces the same scan output for the bundled plugins' constructs" do+ plugin_fixtures = {+ "poll" => "[poll type=regular]\n* one\n* two @someuser\n[/poll]\n\nafter @someuser",+ "discourse-details" => %{[details="Summary"]\nhidden @someuser\n[/details]},+ "spoiler-alert" => "[spoiler]shh @someuser[/spoiler]",+ "footnote" => "noted[^1]\n\n[^1]: the note with @someuser",+ "discourse-local-dates" =>+ %{[date=2026-08-27 time=13:37:00 timezone="Europe/Vienna"] with @someuser},+ "checklist" => "[ ] open task @someuser\n[x] done task",+ "chat" =>+ "[chat quote=\"someuser;123;2026-08-27T00:00:00Z\" channel=\"general\" channelId=\"1\"]\nhi @someuser\n[/chat]",+ }+ loaded = Discourse.plugins.map(&:name)+ testable = plugin_fixtures.select { |plugin, _| loaded.include?(plugin) }+ skip "no bundled plugin is loaded — run with LOAD_PLUGINS=1" if testable.empty?++ posts = testable.values.each_with_index.map { |raw, index| { "id" => index, "raw" => raw } }+ standalone = context.scan(posts.map { |post| { id: post["id"], raw: post["raw"] } })+ booted = pretty_text_scan(posts)++ standalone+ .zip(booted, testable.keys)+ .each { |ours, theirs, plugin| expect(ours).to eq(theirs), <<~MSG }+ scan mismatch for the #{plugin} fixture:+ #{testable[plugin].inspect}++ standalone: #{ours.inspect}+ booted: #{theirs.inspect}+ MSG+ end++ it "builds the same unicode emoji replacement table as the Emoji model" do+ expect(Migrations::Converters::MarkdownEngine::EmojiData.unicode_replacements).to eq(+ JSON.parse(Emoji.unicode_replacements_json),+ )+ end++ # `Bundle` still transpiles plugin markdown features per file through+ # `AssetProcessor`. That is the only reason a cold build boots in-process+ # V8 and therefore runs in a separate build process. When the host one day+ # precompiles plugin features itself, all of that can be deleted — and this+ # example exists because nothing else would fail on that day: our per-file+ # path would keep producing identical output forever.+ it "still has to transpile plugin markdown features itself" do+ plugin_bundles =+ PrettyText.constants.filter_map do |name|+ constant = PrettyText.const_get(name)+ constant if constant.is_a?(PrecompiledBundle)+ end++ # Plugin markdown features live under `plugins/*/…/discourse-markdown/`;+ # the core `discourse-markdown-it` sources deliberately don't match.+ plugin_globs =+ plugin_bundles+ .flat_map(&:dependency_globs)+ .select { |glob| glob.include?("/discourse-markdown/") || glob.start_with?("plugins/") }++ expect(plugin_globs).to be_empty,+ "a PrettyText bundle now covers plugin markdown (#{plugin_globs.inspect}) — " \+ "the engine Bundle's per-file plugin transpiles, build_in_subprocess, and " \+ "the V8 half of HostShims can likely be deleted in favor of it"+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/markdown_engine/runtime_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_engine/runtime_spec.rbnew file mode 100644index 00000000000..44013928a4c--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/markdown_engine/runtime_spec.rb@@ -0,0 +1,38 @@+# frozen_string_literal: true++require "mini_racer"++RSpec.describe Migrations::Converters::MarkdownEngine do+ let(:context) { MiniRacer::Context.new }++ before do+ root = Migrations::Converters::MarkdownEngine.discourse_root+ context.eval(+ File.read(+ File.join(+ root,+ "migrations/converters/lib/migrations/converters/markdown_engine/runtime.js",+ ),+ ),+ )+ context.eval(<<~JS)+ __scanConfig = { categorySlugs: {}, tagNames: { "κοσμος": true, "κοσμοσ": true } };+ function lookup(slug) { return __Ruby.hashtag_lookup(slug, null, ["tag"]); }+ JS+ end++ after { context.dispose }++ it "matches Ruby identity normalization without conflating lowercase sigmas" do+ refs = %w[κοσμος κοσμοσ ΚΟΣΜΟΣ].map { |name| context.call("lookup", name).fetch("ref") }++ expect(refs).to eq(%w[κοσμος κοσμοσ κοσμοσ])+ end++ it "rejects a name when only the other lowercase sigma exists" do+ context.eval('delete __scanConfig.tagNames["κοσμος"];')++ expect(context.call("lookup", "κοσμος")).to be_nil+ expect(context.call("lookup", "ΚΟΣΜΟΣ").fetch("ref")).to eq("κοσμοσ")+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/markdown_renderer_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_renderer_spec.rbindex ea848a6cc5f..5df89873692 100644--- a/migrations/converters/spec/lib/migrations/converters/markdown_renderer_spec.rb+++ b/migrations/converters/spec/lib/migrations/converters/markdown_renderer_spec.rb@@ -63,13 +63,13 @@ RSpec.describe Migrations::Converters::MarkdownRenderer dolet(:link_renderer) { described_class.new(format: :bbcode, embeds: buffer, defer: %i[link]) }- it "defers an attributed quote, recording the linkage and preserving the body" do+ it "defers a quote that carries a source reference, recording the linkage and preserving the body" doraw =renderer.to_markdown('[quote="John, post:12, topic:34, username:john"]quoted body[/quote]')expect(buffer.quotes.size).to eq(1)descriptor = buffer.quotes.first- # The Discourse attribution format carries coordinates: `post:` is a post+ # The Discourse quote header carries coordinates: `post:` is a post# number, `topic:` a topic id.expect(descriptor[:quoted_post_id]).to be_nilexpect(descriptor[:quoted_topic_id]).to eq(34)@@ -85,8 +85,9 @@ RSpec.describe Migrations::Converters::MarkdownRenderer doexpect(raw).to include("[/quote]")end- it "drops an attribution number too large for an id column" do- # Meta really has a post titled like this; SQLite raises binding a bignum.+ it "drops a quote id too large for an id column" do+ # A real forum turned out to have a post titled like this; SQLite raises+ # binding a bignum.raw = renderer.to_markdown('[quote="A, post:77777777777777777789999, topic:2"]q[/quote]')descriptor = buffer.quotes.first@@ -96,7 +97,7 @@ RSpec.describe Migrations::Converters::MarkdownRenderer doexpect(raw).to include(descriptor[:placeholder])end- it "renders an unattributed quote natively (nothing to remap)" do+ it "renders a quote with no header natively (nothing to remap)" doraw = renderer.to_markdown("[quote]just text[/quote]")expect(buffer).to be_empty@@ -300,12 +301,12 @@ RSpec.describe Migrations::Converters::MarkdownRenderer dotoken = extract.call(node, nil)expect(collector.uploads).to contain_exactly(- { placeholder: token, upload_id: "abc123", original_markdown: nil },+ { placeholder: token, upload_id: "abc123", original_markdown: nil, external_host: nil },)endit "maps a Quote node's ids to quoted_post_id and quoted_user_id" do- # BBCode can't carry them (phpBB-style id attribution arrives via the+ # BBCode can't carry them (phpBB-style id references arrive via the# TextFormatter parser), so exercise the method directly._node_class, extract = renderer.embed_handlers.fetch(:quote)node = Markbridge::AST::Quote.new(username: "alice", post_id: 9001, user_id: 12)@@ -398,6 +399,7 @@ RSpec.describe Migrations::Converters::MarkdownRenderer domention_type: Migrations::Database::IntermediateDB::Enums::MentionType::USER,target_id: nil,name: "gerhard",+ original_markdown: nil,},)enddiff --git a/migrations/converters/spec/support/boundary_corpus.rb b/migrations/converters/spec/support/boundary_corpus.rbnew file mode 100644index 00000000000..d5a61818109--- /dev/null+++ b/migrations/converters/spec/support/boundary_corpus.rb@@ -0,0 +1,47 @@+# frozen_string_literal: true++# The boundary characters the construct parity batteries probe, shared so every+# construct is checked against the same set: the 32 ASCII punctuation characters+# (CommonMark), which already include the ASCII symbols `$ + < = > ^ ` | ~`,+# plus letters, whitespace, and the Unicode characters that separate a plain+# "not a word character" boundary from the one core actually applies — the wide+# spaces (NBSP, ideographic space), `²`/`½` (category No), `€` (a currency+# symbol, category Sc), and `` (soft hyphen, category Cf). The invisible ones+# are written as escapes so they cannot silently regress to a plain space.+module BoundaryCorpus+ # @param zero_width_space [Boolean] include U+200B, which core's emoji rule+ # accepts as a boundary although it is neither a space it counts nor+ # punctuation.+ def self.chars(zero_width_space: false)+ ascii_punctuation =+ [0x21..0x2f, 0x3a..0x40, 0x5b..0x60, 0x7b..0x7e].flat_map(&:to_a)+ .to_h { |cp| [format("U+%04X", cp), cp.chr(Encoding::UTF_8)] }++ chars = {+ "letter a" => "a",+ "digit 9" => "9",+ "e-acute" => "é",+ "han" => "漢",+ "space" => " ",+ "tab" => "\t",+ "newline" => "\n",+ "no-break space" => " ",+ "ideographic space" => " ",+ "em dash" => "—",+ "low double quote" => "„",+ "left guillemet" => "«",+ "ellipsis" => "…",+ "euro sign" => "€",+ "superscript two" => "²",+ "vulgar half" => "½",+ "soft hyphen" => "",+ }+ chars["zero-width space"] = "" if zero_width_space+ chars.merge(ascii_punctuation)+ end++ def self.describe(char)+ codepoints = char.each_codepoint.map { |cp| format("U+%04X", cp) }.join(" ")+ "#{char.inspect} (#{codepoints})"+ end+enddiff --git a/migrations/converters/spec/support/markdown_engine_helper.rb b/migrations/converters/spec/support/markdown_engine_helper.rbnew file mode 100644index 00000000000..4f0877b4009--- /dev/null+++ b/migrations/converters/spec/support/markdown_engine_helper.rb@@ -0,0 +1,55 @@+# frozen_string_literal: true++# The extractor requires a markdown engine context, and building one means a+# V8 isolate evaluating the whole engine bundle (~70ms warm) — too much to pay+# per example. Contexts are stateless across scans, so they are memoized per+# configuration for the whole spec run (one isolate is ~33 MiB RSS, so specs+# should reuse the canonical configurations here rather than inventing+# near-duplicates) and disposed together after the suite.+module MarkdownEngineHelper+ def self.bundle+ @bundle ||= Migrations::Converters::MarkdownEngine::Bundle.load_or_build+ end++ def self.close_all+ @contexts&.each_value(&:close)+ @contexts = nil+ end++ def self.context_for(category_slugs: [], tag_names: [], custom_emoji_names: [], settings: {})+ key = [category_slugs.sort, tag_names.sort, custom_emoji_names.sort, settings.sort]+ @contexts ||= {}+ @contexts[key] ||= Migrations::Converters::MarkdownEngine::Context.new(+ bundle:,+ config:+ Migrations::Converters::MarkdownEngine::Config.new(+ source_settings: settings,+ category_slugs:,+ tag_names:,+ custom_emoji_names:,+ ),+ )+ end++ # Runs `js` inside a memoized context's isolate, for a spec that needs the+ # engine's JavaScript surface itself rather than a scan of a body.+ def self.eval_js(js, **config)+ context_for(**config).instance_variable_get(:@context).eval(js)+ end++ # An engine configuration matching a raw-extractor spec's Ruby-side name+ # sets. Whether a slug resolves as a category or a tag does not reach the+ # extracted rows (the forced type comes from the source's `::type` suffix),+ # so every name goes into both name sets — `Config` derives what the+ # engine's lookup needs from them.+ def self.context_for_names(hashtag_names:, custom_emoji_names: [], settings: {})+ context_for(+ category_slugs: hashtag_names,+ tag_names: hashtag_names,+ custom_emoji_names:,+ settings: { "unicode_usernames" => true }.merge(settings),+ )+ end+end++RSpec.configure { |config| config.after(:suite) { MarkdownEngineHelper.close_all } }diff --git a/migrations/converters/spec/support/raw_extractor_context.rb b/migrations/converters/spec/support/raw_extractor_context.rbnew file mode 100644index 00000000000..324ec4d6cfe--- /dev/null+++ b/migrations/converters/spec/support/raw_extractor_context.rb@@ -0,0 +1,123 @@+# frozen_string_literal: true++RSpec.shared_context "with raw extractor" do+ subject(:extractor) do+ described_class.new(+ embeds: buffer,+ mention_names:,+ hashtag_names:,+ markdown_engine:,+ custom_emoji_names:,+ internal_link_hosts:,+ internal_link_base_prefix:,+ on_foreign_host:,+ on_engine_refusal:,+ )+ end++ # Constructor arguments a spec overrides individually instead of rebuilding+ # the whole subject; the defaults mirror the constructor's own.+ let(:custom_emoji_names) { nil }+ let(:internal_link_hosts) { {} }+ let(:internal_link_base_prefix) { nil }+ let(:on_foreign_host) { nil }+ let(:on_engine_refusal) { nil }++ # The host most link specs configure as the source's own.+ let(:source_host) { "forum.example.com" }++ # A valid 40-hex upload SHA1 for upload fixtures.+ let(:sha1) { "0123456789abcdef0123456789abcdef01234567" }++ # Memoized per configuration for the whole run; specs that need different+ # engine-side names override this with another helper call.+ let(:markdown_engine) do+ MarkdownEngineHelper.context_for_names(hashtag_names: raw_hashtag_name_list)+ end++ # The extractor defers a mention only when the source has that name, so the+ # names a spec mentions have to exist. Override for a spec about the gate+ # itself.+ let(:mention_names) do+ Migrations::CompactStringSet.new(+ %w[+ admins+ alice+ all+ bob+ carol+ café_team+ channel+ gerhard+ here+ j-d+ john+ john.doe+ real+ some-user+ someuser+ staff+ user_+ José+ 田中+ ].map { |name| Migrations::NameNormalizer.normalize(name) },+ )+ end++ # Same for hashtags: the source has to have the category or tag. `unknown` is+ # deliberately absent — specs use it for the name that resolves to nothing.+ # The plain list also feeds the engine context's lookup sets, which decide+ # whether a `#slug` produces a token at all.+ let(:raw_hashtag_name_list) do+ %w[announcements general news release reply support support:billing team v2.0]+ end++ let(:hashtag_names) do+ Migrations::CompactStringSet.new(+ raw_hashtag_name_list.map { |name| Migrations::NameNormalizer.normalize(name) },+ )+ end++ # The intermediate DB enums, referenced through the namespace so a call site+ # names the enum it means: `enums::MentionType::USER`.+ let(:enums) { Migrations::Database::IntermediateDB::Enums }++ let(:buffer) do+ Migrations::Converters::EmbedBuffer.new(+ owner_type: Migrations::Database::IntermediateDB::Enums::EmbedOwner::POST,+ )+ end++ def extract(raw, topic_id: nil)+ extractor.extract(raw, topic_id:)+ end++ def link_for(raw)+ result = extract(raw)+ [buffer.links.first, result]+ end+end++# Scaffolding for the `:rails` parity batteries. They compare one construct+# against `PrettyText.cook` row by row, so each row needs its own buffer rather+# than the shared subject above.+RSpec.shared_context "with parity extractor" do+ # A battery is about one construct; the extractor requires both name sets+ # anyway, and an empty one defers nothing. A spec whose construct is gated on+ # a name overrides the set it needs.+ let(:mention_names) { Migrations::CompactStringSet.new([]) }+ let(:hashtag_names) { Migrations::CompactStringSet.new([]) }+ let(:markdown_engine) { MarkdownEngineHelper.context_for_names(hashtag_names: []) }++ def new_buffer+ Migrations::Converters::EmbedBuffer.new(+ owner_type: Migrations::Database::IntermediateDB::Enums::EmbedOwner::POST,+ )+ end++ def build_extractor(buffer, **overrides)+ described_class.new(+ **{ embeds: buffer, mention_names:, hashtag_names:, markdown_engine: }.merge(overrides),+ )+ end+enddiff --git a/migrations/core/db/intermediate_db_schema/100-base-schema.sql b/migrations/core/db/intermediate_db_schema/100-base-schema.sqlindex 6a4cd0da2a6..5e002b838b7 100644--- a/migrations/core/db/intermediate_db_schema/100-base-schema.sql+++ b/migrations/core/db/intermediate_db_schema/100-base-schema.sql@@ -139,18 +139,21 @@ CREATE INDEX idx_embed_events_owner_type_owner_id ON embed_events (owner_type, oCREATE TABLE embed_hashtags(- hashtag_type ENUM_INTEGER,- name TEXT NOT NULL,- owner_id NUMERIC NOT NULL,- owner_type ENUM_INTEGER NOT NULL,- placeholder TEXT NOT NULL,- target_id NUMERIC+ hashtag_type ENUM_INTEGER,+ name TEXT NOT NULL,+ original_markdown TEXT,+ owner_id NUMERIC NOT NULL,+ owner_type ENUM_INTEGER NOT NULL,+ placeholder TEXT NOT NULL,+ target_id NUMERIC);CREATE INDEX idx_embed_hashtags_owner_type_owner_id ON embed_hashtags (owner_type, owner_id);CREATE TABLE embed_links(+ label_url_offset INTEGER,+ original_markdown TEXT,owner_id NUMERIC NOT NULL,owner_type ENUM_INTEGER NOT NULL,placeholder TEXT NOT NULL,@@ -158,22 +161,25 @@ CREATE TABLE embed_linkstarget_name TEXT,target_post_number INTEGER,target_suffix TEXT,+ target_tag_path TEXT,target_topic_id NUMERIC,target_type ENUM_INTEGER,text TEXT,- url TEXT+ url TEXT,+ url_offset INTEGER);CREATE INDEX idx_embed_links_owner_type_owner_id ON embed_links (owner_type, owner_id);CREATE TABLE embed_mentions(- mention_type ENUM_INTEGER,- name TEXT,- owner_id NUMERIC NOT NULL,- owner_type ENUM_INTEGER NOT NULL,- placeholder TEXT NOT NULL,- target_id NUMERIC+ mention_type ENUM_INTEGER,+ name TEXT,+ original_markdown TEXT,+ owner_id NUMERIC NOT NULL,+ owner_type ENUM_INTEGER NOT NULL,+ placeholder TEXT NOT NULL,+ target_id NUMERIC);CREATE INDEX idx_embed_mentions_owner_type_owner_id ON embed_mentions (owner_type, owner_id);@@ -190,6 +196,7 @@ CREATE INDEX idx_embed_polls_owner_type_owner_id ON embed_polls (owner_type, ownCREATE TABLE embed_quotes(+ original_markdown TEXT,owner_id NUMERIC NOT NULL,owner_type ENUM_INTEGER NOT NULL,placeholder TEXT NOT NULL,@@ -205,6 +212,7 @@ CREATE INDEX idx_embed_quotes_owner_type_owner_id ON embed_quotes (owner_type, oCREATE TABLE embed_uploads(+ external_host TEXT,original_markdown TEXT,owner_id NUMERIC NOT NULL,owner_type ENUM_INTEGER NOT NULL,@@ -410,6 +418,7 @@ CREATE TABLE topicspinned_globally BOOLEAN,pinned_until DATETIME,slow_mode_seconds INTEGER,+ slug TEXT,subtype TEXT,title TEXT NOT NULL,user_id NUMERIC,@@ -419,6 +428,7 @@ CREATE TABLE topics);CREATE INDEX idx_topics_archetype ON topics (archetype);+CREATE INDEX idx_topics_slug ON topics (slug);CREATE TABLE uploads(diff --git a/migrations/core/lib/migrations/compact_string_set.rb b/migrations/core/lib/migrations/compact_string_set.rbnew file mode 100644index 00000000000..412057a6126--- /dev/null+++ b/migrations/core/lib/migrations/compact_string_set.rb@@ -0,0 +1,214 @@+# frozen_string_literal: true++module Migrations+ # An immutable membership set for a large number of short strings (usernames,+ # tag names) whose bulk lives in malloc'd buffers the GC never writes into, so+ # it stays copy-on-write-stable across forks.+ #+ # A step's args are built after the scheduler's one-time `Process.warmup`, so+ # their objects are still young when the step's workers fork. Ruby's+ # generational GC then stamps age bits into every live object's slot during+ # the children's early GC cycles, and that write privatizes the copy-on-write+ # page the slot sits on. A `Set` of a million usernames is a million such+ # slots (names of 23 bytes or fewer are embedded, so their characters live in+ # the slot too) — on the order of 80-110 MB that every fork ends up copying.+ #+ # Here the names are held as a few frozen Strings: the name bytes concatenated+ # into one buffer, their byte offsets packed into another, and an+ # open-addressing probe table of packed 32-bit fingerprints beside them. That+ # is a fixed number of heap slots regardless of the name count.+ #+ # Membership is a hash probe: `String#hash` picks the slot, the packed+ # fingerprint filters non-members without touching the name bytes, and only a+ # fingerprint hit pays for one `byteslice` to confirm — so membership stays+ # exact, never probabilistic. Forked children inherit the parent's+ # `String#hash` seed, so a table built before fork answers after it.+ #+ # Construction streams: `names` is consumed once and duplicates are dropped by+ # the same probe the lookups use, so the build-time peak is just the buffers.+ class CompactStringSet+ # Offsets and probe entries are 32-bit, so the name buffer must stay+ # addressable in 32 bits.+ MAX_TOTAL_BYTES = (2**32) - 1+ private_constant :MAX_TOTAL_BYTES++ # A stored fingerprint of 0 marks an empty probe slot; a real fingerprint+ # that hashes to 0 is nudged to 1, which only widens that fingerprint's+ # filter by one value in four billion.+ EMPTY = 0+ private_constant :EMPTY++ # Above this share of filled slots the table doubles; at 50% the expected+ # probe sequence for a miss stays around two slots.+ MAX_LOAD = 0.5+ private_constant :MAX_LOAD++ # @param names [Enumerable<String>] the members; deduped here, consumed+ # once.+ def initialize(names)+ # The name buffer must be UTF-8 (`String.new` defaults to binary): the+ # confirm slices compare against UTF-8 queries, and equal bytes in+ # incompatible encodings are not equal strings.+ @count = 0+ @max_byte_length = 0+ buffer = String.new(capacity: 4096, encoding: Encoding::UTF_8)+ offsets = String.new(capacity: 1024, encoding: Encoding::BINARY)+ offsets << [0].pack("V")++ @capacity = 1024+ @mask = @capacity - 1+ table = empty_table(@capacity)++ names.each do |name|+ # Membership before growth, so a duplicate arriving exactly at the+ # threshold does not double the table for nothing.+ next if build_member?(table, buffer, offsets, name)++ table = grow(table, buffer, offsets) if @count >= @capacity * MAX_LOAD+ claim_absent(table, buffer, offsets, name)+ end++ @table = exact_copy(table, Encoding::BINARY)+ @buffer = exact_copy(buffer, Encoding::UTF_8)+ @offsets = exact_copy(offsets, Encoding::BINARY)+ end++ def include?(name)+ hash = name.hash+ fingerprint = fingerprint_for(hash)+ index = hash & @mask++ # `Kernel#loop` allocates per call; the plain `while` keeps a miss+ # allocation-free.+ stored = read32(@table, index * 8)+ while stored != EMPTY+ return true if stored == fingerprint && entry_at(index) == name++ index = (index + 1) & @mask+ stored = read32(@table, index * 8)+ end++ false+ end++ def size+ @count+ end++ # The longest member in bytes, so a scanner deciding whether a run of raw+ # bytes could spell a member reads no more than this many.+ attr_reader :max_byte_length++ private++ # The slot index comes from the hash's low bits, so the fingerprint takes+ # bits from further up to stay informative within a probe bucket.+ def fingerprint_for(hash)+ fingerprint = (hash >> 27) & 0xffff_ffff+ fingerprint == EMPTY ? 1 : fingerprint+ end++ # A probe slot is 8 packed bytes: the fingerprint, then the member's index+ # into the offsets buffer.+ def slot(table, index)+ [table.unpack1("V", offset: index * 8), table.unpack1("V", offset: index * 8 + 4)]+ end++ def entry_at(probe_index)+ member = read32(@table, probe_index * 8 + 4)+ start = read32(@offsets, member * 4)+ finish = read32(@offsets, (member + 1) * 4)+ @buffer.byteslice(start, finish - start)+ end++ # `unpack1` allocates one object per call, so the probe path assembles its+ # little-endian 32-bit reads from `getbyte`.+ def read32(string, byte_offset)+ string.getbyte(byte_offset) | (string.getbyte(byte_offset + 1) << 8) |+ (string.getbyte(byte_offset + 2) << 16) | (string.getbyte(byte_offset + 3) << 24)+ end++ def build_member?(table, buffer, offsets, name)+ hash = name.hash+ fingerprint = fingerprint_for(hash)+ index = hash & @mask++ loop do+ stored, member = slot(table, index)+ return false if stored == EMPTY++ if stored == fingerprint+ start = offsets.unpack1("V", offset: member * 4)+ finish = offsets.unpack1("V", offset: (member + 1) * 4)+ return true if buffer.byteslice(start, finish - start) == name+ end++ index = (index + 1) & @mask+ end+ end++ # The caller established the name is absent, so no equality checks are+ # needed on the way.+ def claim_absent(table, buffer, offsets, name)+ hash = name.hash+ index = hash & @mask+ index = (index + 1) & @mask while slot(table, index).first != EMPTY++ claim(table, index, fingerprint_for(hash), buffer, offsets, name)+ end++ def claim(table, index, fingerprint, buffer, offsets, name)+ position = buffer.bytesize + name.bytesize+ if position > MAX_TOTAL_BYTES+ raise ArgumentError,+ "name bytes exceed the set's 32-bit addressable capacity " \+ "(#{MAX_TOTAL_BYTES} bytes)"+ end++ table.bytesplice(index * 8, 8, [fingerprint, @count].pack("VV"))+ buffer << name+ offsets << [position].pack("V")+ @count += 1+ @max_byte_length = name.bytesize if name.bytesize > @max_byte_length+ end++ # The name and offset buffers only ever append, so they carry over+ # untouched.+ def grow(table, buffer, offsets)+ @capacity *= 2+ @mask = @capacity - 1+ grown = empty_table(@capacity)++ each_member(buffer, offsets) do |member, name|+ hash = name.hash+ fingerprint = fingerprint_for(hash)+ index = hash & @mask+ index = (index + 1) & @mask while grown.unpack1("V", offset: index * 8) != EMPTY+ grown.bytesplice(index * 8, 8, [fingerprint, member].pack("VV"))+ end++ grown+ end++ def each_member(buffer, offsets)+ start = 0+ @count.times do |member|+ finish = offsets.unpack1("V", offset: (member + 1) * 4)+ yield member, buffer.byteslice(start, finish - start)+ start = finish+ end+ end++ def empty_table(capacity)+ String.new("\0" * (capacity * 8), encoding: Encoding::BINARY)+ end++ # Appending leaves malloc capacity of roughly twice the content; an+ # exact-capacity copy sheds that slack for the structure's whole lifetime.+ def exact_copy(string, encoding)+ copy = String.new(capacity: string.bytesize, encoding:)+ copy << string+ copy.freeze+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_hashtag.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_hashtag.rbindex 299bdde31fa..9f77ef3c2b8 100644--- a/migrations/core/lib/migrations/database/intermediate_db/embed_hashtag.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_hashtag.rb@@ -12,27 +12,29 @@ module MigrationsINSERT INTO embed_hashtags (hashtag_type,name,+ original_markdown,owner_id,owner_type,placeholder,target_id)VALUES (- ?, ?, ?, ?, ?, ?+ ?, ?, ?, ?, ?, ?, ?)SQLprivate_constant :SQL# Creates a new `embed_hashtags` record in the IntermediateDB.#- # @param hashtag_type [Integer, nil]+ # @param hashtag_type [Integer, nil]# Any constant from HashtagType (e.g. HashtagType::CATEGORY)- # @param name [String]- # @param owner_id [Integer, String]- # @param owner_type [Integer]+ # @param name [String]+ # @param original_markdown [String, nil]+ # @param owner_id [Integer, String]+ # @param owner_type [Integer]# Any constant from EmbedOwner (e.g. EmbedOwner::POST)- # @param placeholder [String]- # @param target_id [Integer, String, nil]+ # @param placeholder [String]+ # @param target_id [Integer, String, nil]## @return [void]#@@ -41,6 +43,7 @@ module Migrationsdef self.create(hashtag_type: nil,name:,+ original_markdown: nil,owner_id:,owner_type:,placeholder:,@@ -50,6 +53,7 @@ module MigrationsSQL,hashtag_type,name,+ original_markdown,owner_id,owner_type,placeholder,diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_link.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_link.rbindex 7d8dd16e9d8..c666e87548c 100644--- a/migrations/core/lib/migrations/database/intermediate_db/embed_link.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_link.rb@@ -10,6 +10,8 @@ module Migrationsmodule EmbedLinkSQL = <<~SQLINSERT INTO embed_links (+ label_url_offset,+ original_markdown,owner_id,owner_type,placeholder,@@ -17,19 +19,23 @@ module Migrationstarget_name,target_post_number,target_suffix,+ target_tag_path,target_topic_id,target_type,text,- url+ url,+ url_offset)VALUES (- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)SQLprivate_constant :SQL# Creates a new `embed_links` record in the IntermediateDB.#+ # @param label_url_offset [Integer, nil]+ # @param original_markdown [String, nil]# @param owner_id [Integer, String]# @param owner_type [Integer]# Any constant from EmbedOwner (e.g. EmbedOwner::POST)@@ -38,17 +44,21 @@ module Migrations# @param target_name [String, nil]# @param target_post_number [Integer, nil]# @param target_suffix [String, nil]+ # @param target_tag_path [String, nil]# @param target_topic_id [Integer, String, nil]# @param target_type [Integer, nil]# Any constant from LinkTarget (e.g. LinkTarget::TOPIC)# @param text [String, nil]# @param url [String, nil]+ # @param url_offset [Integer, nil]## @return [void]## @see Migrations::Database::IntermediateDB::Enums::EmbedOwner# @see Migrations::Database::IntermediateDB::Enums::LinkTargetdef self.create(+ label_url_offset: nil,+ original_markdown: nil,owner_id:,owner_type:,placeholder:,@@ -56,13 +66,17 @@ module Migrationstarget_name: nil,target_post_number: nil,target_suffix: nil,+ target_tag_path: nil,target_topic_id: nil,target_type: nil,text: nil,- url: nil+ url: nil,+ url_offset: nil)Migrations::Database::IntermediateDB.insert(SQL,+ label_url_offset,+ original_markdown,owner_id,owner_type,placeholder,@@ -70,10 +84,12 @@ module Migrationstarget_name,target_post_number,target_suffix,+ target_tag_path,target_topic_id,target_type,text,url,+ url_offset,)endenddiff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_mention.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_mention.rbindex 239d60be47f..669b0035469 100644--- a/migrations/core/lib/migrations/database/intermediate_db/embed_mention.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_mention.rb@@ -12,27 +12,29 @@ module MigrationsINSERT INTO embed_mentions (mention_type,name,+ original_markdown,owner_id,owner_type,placeholder,target_id)VALUES (- ?, ?, ?, ?, ?, ?+ ?, ?, ?, ?, ?, ?, ?)SQLprivate_constant :SQL# Creates a new `embed_mentions` record in the IntermediateDB.#- # @param mention_type [Integer, nil]+ # @param mention_type [Integer, nil]# Any constant from MentionType (e.g. MentionType::USER)- # @param name [String, nil]- # @param owner_id [Integer, String]- # @param owner_type [Integer]+ # @param name [String, nil]+ # @param original_markdown [String, nil]+ # @param owner_id [Integer, String]+ # @param owner_type [Integer]# Any constant from EmbedOwner (e.g. EmbedOwner::POST)- # @param placeholder [String]- # @param target_id [Integer, String, nil]+ # @param placeholder [String]+ # @param target_id [Integer, String, nil]## @return [void]#@@ -41,6 +43,7 @@ module Migrationsdef self.create(mention_type: nil,name: nil,+ original_markdown: nil,owner_id:,owner_type:,placeholder:,@@ -50,6 +53,7 @@ module MigrationsSQL,mention_type,name,+ original_markdown,owner_id,owner_type,placeholder,diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_quote.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_quote.rbindex 56c09c45696..87afbeff5e1 100644--- a/migrations/core/lib/migrations/database/intermediate_db/embed_quote.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_quote.rb@@ -10,6 +10,7 @@ module Migrationsmodule EmbedQuoteSQL = <<~SQLINSERT INTO embed_quotes (+ original_markdown,owner_id,owner_type,placeholder,@@ -21,13 +22,14 @@ module Migrationsquoted_username)VALUES (- ?, ?, ?, ?, ?, ?, ?, ?, ?+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)SQLprivate_constant :SQL# Creates a new `embed_quotes` record in the IntermediateDB.#+ # @param original_markdown [String, nil]# @param owner_id [Integer, String]# @param owner_type [Integer]# Any constant from EmbedOwner (e.g. EmbedOwner::POST)@@ -43,6 +45,7 @@ module Migrations## @see Migrations::Database::IntermediateDB::Enums::EmbedOwnerdef self.create(+ original_markdown: nil,owner_id:,owner_type:,placeholder:,@@ -55,6 +58,7 @@ module Migrations)Migrations::Database::IntermediateDB.insert(SQL,+ original_markdown,owner_id,owner_type,placeholder,diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_upload.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_upload.rbindex c47a95f411b..9b07a1578be 100644--- a/migrations/core/lib/migrations/database/intermediate_db/embed_upload.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_upload.rb@@ -10,6 +10,7 @@ module Migrationsmodule EmbedUploadSQL = <<~SQLINSERT INTO embed_uploads (+ external_host,original_markdown,owner_id,owner_type,@@ -17,13 +18,14 @@ module Migrationsupload_id)VALUES (- ?, ?, ?, ?, ?+ ?, ?, ?, ?, ?, ?)SQLprivate_constant :SQL# Creates a new `embed_uploads` record in the IntermediateDB.#+ # @param external_host [String, nil]# @param original_markdown [String, nil]# @param owner_id [Integer, String]# @param owner_type [Integer]@@ -35,6 +37,7 @@ module Migrations## @see Migrations::Database::IntermediateDB::Enums::EmbedOwnerdef self.create(+ external_host: nil,original_markdown: nil,owner_id:,owner_type:,@@ -43,6 +46,7 @@ module Migrations)Migrations::Database::IntermediateDB.insert(SQL,+ external_host,original_markdown,owner_id,owner_type,diff --git a/migrations/core/lib/migrations/database/intermediate_db/enums/link_target.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/link_target.rbindex d9d8360510f..277a8aea94b 100644--- a/migrations/core/lib/migrations/database/intermediate_db/enums/link_target.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/link_target.rb@@ -18,6 +18,9 @@ module MigrationsTAG = 5GROUP = 6BADGE = 7+ SITE = 8+ CATEGORY_TAG = 9+ TAG_INTERSECTION = 10endendenddiff --git a/migrations/core/lib/migrations/database/intermediate_db/topic.rb b/migrations/core/lib/migrations/database/intermediate_db/topic.rbindex 84ecc3ec057..f4093344b1b 100644--- a/migrations/core/lib/migrations/database/intermediate_db/topic.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/topic.rb@@ -27,6 +27,7 @@ module Migrationspinned_globally,pinned_until,slow_mode_seconds,+ slug,subtype,title,user_id,@@ -35,7 +36,7 @@ module Migrationsvisible)VALUES (- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)SQLprivate_constant :SQL@@ -59,6 +60,7 @@ module Migrations# @param pinned_globally [Boolean, nil]# @param pinned_until [Time, nil]# @param slow_mode_seconds [Integer, nil]+ # @param slug [String, nil]# @param subtype [String, nil]# @param title [String]# @param user_id [Integer, String, nil]@@ -85,6 +87,7 @@ module Migrationspinned_globally: nil,pinned_until: nil,slow_mode_seconds: nil,+ slug: nil,subtype: nil,title:,user_id: nil,@@ -111,6 +114,7 @@ module MigrationsMigrations::Database.format_boolean(pinned_globally),Migrations::Database.format_datetime(pinned_until),slow_mode_seconds,+ slug,subtype,title,user_id,diff --git a/migrations/core/lib/migrations/name_normalizer.rb b/migrations/core/lib/migrations/name_normalizer.rbindex b5d36e8f781..7803a9acf4b 100644--- a/migrations/core/lib/migrations/name_normalizer.rb+++ b/migrations/core/lib/migrations/name_normalizer.rb@@ -1,13 +1,14 @@# frozen_string_literal: truemodule Migrations- # Normalizes a username or group name for comparison the way Discourse does it- # (Unicode NFC, then downcase), so a mention and the user or group it names match- # however the source encoded them. Shared by the converter that types a mention- # (`Converters::Discourse::MentionClassifier`) and the importer that maps a recorded- # name back to a user or group (`Importer::PlaceholderResolver`), so the two sides- # can't disagree on what counts as the same name.+ # Normalizes a username, group name, tag or category slug for comparison the+ # way Discourse does it (Unicode NFC, then downcase), so a mention and the+ # user or group it names match however the source encoded them. Shared by the+ # converter and the importer, so the two sides can't disagree on what counts+ # as the same name.module NameNormalizer+ # Lowercase final and medial sigmas are distinct identity keys in core.+ # Broader comparisons for locating markdown tokens belong in the scanner.def self.normalize(name)name.unicode_normalize.downcaseenddiff --git a/migrations/core/spec/lib/common/reporting/plain_spec.rb b/migrations/core/spec/lib/common/reporting/plain_spec.rbindex 76a72f99beb..9ed029ad2dd 100644--- a/migrations/core/spec/lib/common/reporting/plain_spec.rb+++ b/migrations/core/spec/lib/common/reporting/plain_spec.rb@@ -22,7 +22,7 @@ RSpec.describe Migrations::Reporting::Plain doexpect(lines.last).to match(/\A✓ Categories 4,281 \(0:12\)\z/)end- it "prints notices indented and attributed to their step" do+ it "prints notices indented under their step" dostep = reporter.start_step("Posts")step.notice("Calculating items took 6 seconds")expect(lines).to include(" Posts Calculating items took 6 seconds")diff --git a/migrations/core/spec/lib/common/reporting/tui_spec.rb b/migrations/core/spec/lib/common/reporting/tui_spec.rbindex b1abc9d4d43..22dc8097c38 100644--- a/migrations/core/spec/lib/common/reporting/tui_spec.rb+++ b/migrations/core/spec/lib/common/reporting/tui_spec.rb@@ -357,7 +357,7 @@ RSpec.describe Migrations::Reporting::Tui doexpect(notice).to start_with("Posts")end- it "renders an unattributed notice when there is no step title" do+ it "renders a notice with no owning step" dorenderer.apply([:notice, nil, "Run starting"])renderer.repaintexpect(screen.content_rows).to include("Run starting")diff --git a/migrations/core/spec/lib/compact_string_set_spec.rb b/migrations/core/spec/lib/compact_string_set_spec.rbnew file mode 100644index 00000000000..6e4df4ae3c9--- /dev/null+++ b/migrations/core/spec/lib/compact_string_set_spec.rb@@ -0,0 +1,94 @@+# frozen_string_literal: true++RSpec.describe Migrations::CompactStringSet do+ it "handles an empty set" do+ set = described_class.new([])++ expect(set.size).to eq(0)+ expect(set.max_byte_length).to eq(0)+ expect(set.include?("anything")).to be false+ end++ it "handles a single entry" do+ set = described_class.new(["alice"])++ expect(set.size).to eq(1)+ expect(set.include?("alice")).to be true+ expect(set.include?("bob")).to be false+ end++ it "answers membership hits and misses" do+ set = described_class.new(%w[carol alice bob dave])++ expect(set.size).to eq(4)+ expect(set.include?("alice")).to be true+ expect(set.include?("bob")).to be true+ expect(set.include?("carol")).to be true+ expect(set.include?("dave")).to be true+ expect(set.include?("erin")).to be false+ expect(set.include?("")).to be false+ end++ it "does not match a strict prefix of a stored name, or a name a stored one is a prefix of" do+ set = described_class.new(["alice"])++ expect(set.include?("ali")).to be false+ expect(set.include?("alicent")).to be false+ end++ it "stores and matches unicode names" do+ set = described_class.new(%w[café 田中 josé_team])++ expect(set.include?("café")).to be true+ expect(set.include?("田中")).to be true+ expect(set.include?("josé_team")).to be true+ expect(set.include?("cafe")).to be false+ end++ it "dedupes duplicate inputs" do+ set = described_class.new(%w[alice alice bob alice bob])++ expect(set.size).to eq(2)+ expect(set.include?("alice")).to be true+ expect(set.include?("bob")).to be true+ end++ it "reports the longest member in bytes, counting a multibyte name's bytes" do+ expect(described_class.new(%w[bob alice]).max_byte_length).to eq(5)+ # "café" is 5 bytes, one more than its 4 characters.+ expect(described_class.new(%w[bob café]).max_byte_length).to eq(5)+ expect(described_class.new(%w[alice alice]).max_byte_length).to eq(5)+ end++ it "answers correctly across probe-table growth" do+ names = Array.new(5_000) { |i| "user_#{i}" }+ set = described_class.new(names)++ expect(set.size).to eq(5_000)+ expect(names.all? { |name| set.include?(name) }).to be true+ expect(set.include?("user_5000")).to be false+ expect(set.include?("user")).to be false+ end++ it "consumes a streaming enumerable once, without materializing it" do+ passes = 0+ names =+ Enumerator.new do |yielder|+ passes += 1+ 1_000.times { |i| yielder << "name_#{i}" }+ end++ set = described_class.new(names)++ expect(passes).to eq(1)+ expect(set.size).to eq(1_000)+ expect(set.include?("name_999")).to be true+ end++ it "matches a query produced as a byteslice of a larger string" do+ set = described_class.new(["bob"])+ query = "say @bob now".byteslice(5, 3)++ expect(set.include?(query)).to be true+ end+enddiff --git a/migrations/core/spec/lib/name_normalizer_spec.rb b/migrations/core/spec/lib/name_normalizer_spec.rbnew file mode 100644index 00000000000..182628295f3--- /dev/null+++ b/migrations/core/spec/lib/name_normalizer_spec.rb@@ -0,0 +1,13 @@+# frozen_string_literal: true++RSpec.describe Migrations::NameNormalizer do+ it "composes and downcases" do+ expect(described_class.normalize("Café")).to eq("café")+ end++ it "preserves the distinct sigma identity keys used by core" do+ expect(%w[ΚΟΣΜΟΣ κοσμος κοσμοσ].map { |name| described_class.normalize(name) }).to eq(+ %w[κοσμοσ κοσμος κοσμοσ],+ )+ end+enddiff --git a/migrations/core/spec/spec_setup.rb b/migrations/core/spec/spec_setup.rbindex ae1350ba201..052197ee008 100644--- a/migrations/core/spec/spec_setup.rb+++ b/migrations/core/spec/spec_setup.rb@@ -49,9 +49,28 @@ module MigrationsSpecSetup# boot the host harness with the cwd at the application root — the same way the# `disco` binary does before loading the Rails environment.def self.boot_rails(spec_dir)+ # A dev container may default RAILS_ENV to development, and `rails_helper`+ # only fills it in when unset. Booting anything but test leaves the test+ # gem group unrequired, which surfaces much later as a missing constant+ # from a support file rather than as a wrong-environment error.+ ENV["RAILS_ENV"] = "test"+rails_root = File.expand_path("../../..", spec_dir)RSpec.configuration.files_to_run- Dir.chdir(rails_root) { require File.join(rails_root, "spec", "rails_helper") }+ Dir.chdir(rails_root) do+ require File.join(rails_root, "spec", "rails_helper")+ warm_asset_processor+ end+ end++ # `AssetProcessor` locates its inputs with a relative glob, so it only builds+ # while the cwd is the application root — and it builds lazily, at the first+ # `PrettyText.cook`, which happens deep inside an example with the cwd back at+ # the gem. Building it here, where the cwd is still right, leaves the memoized+ # context for those examples to use. Cheap after the first run: the built+ # processor is cached on disk.+ def self.warm_asset_processor+ PrettyText.cook("warm up the markdown pipeline")enddef self.load_support(dir)diff --git a/migrations/importer/lib/migrations/importer/placeholder_linkages.rb b/migrations/importer/lib/migrations/importer/placeholder_linkages.rbnew file mode 100644index 00000000000..dc4ba05cba1--- /dev/null+++ b/migrations/importer/lib/migrations/importer/placeholder_linkages.rb@@ -0,0 +1,237 @@+# frozen_string_literal: true++module Migrations+ module Importer+ # Loads a batch's embed rows and resolves source names and coordinates to+ # IntermediateDB original ids. Rendering is deliberately separate: this is+ # the only component in placeholder substitution that queries the database.+ class PlaceholderLinkages+ include TagLinkPath++ # SQLite builds may accept more variables, but 999 is the historical+ # guaranteed ceiling. Keeping the limit here makes every dynamic IN query+ # safe independently of the caller's owner batch size.+ DEFAULT_BIND_LIMIT = 999++ TABLES = {+ quote: "embed_quotes",+ link: "embed_links",+ mention: "embed_mentions",+ hashtag: "embed_hashtags",+ emoji: "embed_emojis",+ poll: "embed_polls",+ event: "embed_events",+ upload: "embed_uploads",+ }.freeze+ private_constant :TABLES++ Enums = Migrations::Database::IntermediateDB::Enums+ private_constant :Enums++ def initialize(intermediate_db, bind_limit: DEFAULT_BIND_LIMIT)+ raise ArgumentError, "bind_limit must be at least 2" if bind_limit < 2++ @intermediate_db = intermediate_db+ @bind_limit = bind_limit+ @names = NameResolver.new(intermediate_db)+ end++ # Returns linkage rows grouped by owner id after filling every source id+ # that can be derived from names or coordinates.+ def load_and_resolve(owner_ids, owner_type:)+ linkages = load_linkages(owner_ids.uniq, owner_type:)+ resolve_linkage_ids(linkages)+ linkages+ end++ private++ def load_linkages(owner_ids, owner_type:)+ buckets = {}+ return buckets if owner_ids.empty?++ # One bind is reserved for owner_type.+ owner_ids.each_slice(@bind_limit - 1) do |slice|+ bind_params = (["?"] * slice.size).join(", ")++ TABLES.each do |kind, table|+ sql = "SELECT * FROM #{table} WHERE owner_type = ? AND owner_id IN (#{bind_params})"+ @intermediate_db.query(sql, owner_type, *slice) do |row|+ (buckets[row[:owner_id]] ||= []) << [kind, row]+ end+ end+ end++ buckets+ end++ def resolve_linkage_ids(linkages)+ rows_by_kind = Hash.new { |hash, kind| hash[kind] = [] }+ linkages.each_value { |pairs| pairs.each { |kind, row| rows_by_kind[kind] << row } }++ fill_post_ids(+ rows_by_kind[:quote],+ id: :quoted_post_id,+ topic: :quoted_topic_id,+ number: :quoted_post_number,+ )+ resolve_quoted_usernames(rows_by_kind[:quote])+ resolve_mention_names(rows_by_kind[:mention])+ resolve_hashtags(rows_by_kind[:hashtag])+ resolve_links(rows_by_kind[:link])+ end++ def fill_post_ids(rows, id:, topic:, number:)+ pending = rows.select { |row| row[id].nil? && row[topic] && row[number] }+ return if pending.empty?++ post_ids = post_ids_for_coordinates(pending.map { |row| [row[topic], row[number]] })+ pending.each { |row| row[id] = post_ids[[row[topic], row[number]]] }+ end++ def post_ids_for_coordinates(coordinates)+ post_ids = {}++ coordinates+ .uniq+ .each_slice(@bind_limit / 2) do |slice|+ values = (["(?, ?)"] * slice.size).join(", ")+ sql = <<~SQL+ SELECT original_id, topic_id, post_number+ FROM posts+ WHERE (topic_id, post_number) IN (VALUES #{values})+ SQL++ @intermediate_db.query(sql, *slice.flatten) do |row|+ post_ids[[row[:topic_id], row[:post_number]]] = row[:original_id]+ end+ end++ post_ids+ end++ def resolve_links(link_rows)+ fill_post_ids(+ link_rows,+ id: :target_id,+ topic: :target_topic_id,+ number: :target_post_number,+ )+ resolve_link_names(link_rows)+ resolve_topic_slugs(link_rows)+ resolve_link_tag_paths(link_rows)+ end++ def resolve_topic_slugs(link_rows)+ pending =+ link_rows.select do |row|+ row[:target_type] == Enums::LinkTarget::TOPIC && row[:target_id].nil? &&+ row[:target_name].present?+ end+ return if pending.empty?++ topic_ids = topic_ids_for_slugs(pending.map { |row| row[:target_name] })+ pending.each { |row| row[:target_id] = topic_ids[row[:target_name]] }+ end++ def topic_ids_for_slugs(slugs)+ topic_ids = {}++ slugs+ .uniq+ .each_slice(@bind_limit) do |slice|+ placeholders = (["?"] * slice.size).join(", ")+ sql = <<~SQL+ SELECT slug, MIN(original_id) AS original_id+ FROM topics+ WHERE slug IN (#{placeholders})+ GROUP BY slug+ HAVING COUNT(*) = 1+ SQL++ @intermediate_db.query(sql, *slice) { |row| topic_ids[row[:slug]] = row[:original_id] }+ end++ topic_ids+ end++ def resolve_link_tag_paths(link_rows)+ link_rows.each do |row|+ next unless multi_tag_link?(row)++ ids = tag_path_tags(row).map { |name| @names.tag_id(name) }+ row[:resolved_tag_ids] = ids.any? && ids.all? ? ids : nil+ end+ end++ def resolve_link_names(link_rows)+ link_rows.each do |row|+ next if row[:target_id] || row[:target_name].blank?++ row[:target_id] = lookup_target_id(row[:target_type], row[:target_name])+ end+ end++ def lookup_target_id(target_type, name)+ case target_type+ when Enums::LinkTarget::USER+ @names.user_id(name)+ when Enums::LinkTarget::GROUP+ @names.group_id(name)+ when Enums::LinkTarget::TAG+ @names.tag_id(name)+ when Enums::LinkTarget::CATEGORY, Enums::LinkTarget::CATEGORY_TAG+ @names.category_id(name)+ end+ end++ def resolve_quoted_usernames(quote_rows)+ quote_rows.each do |row|+ next if row[:quoted_user_id] || row[:quoted_username].blank?++ row[:quoted_user_id] = @names.user_id(row[:quoted_username])+ end+ end++ def resolve_mention_names(mention_rows)+ mention_rows.each do |row|+ next if row[:target_id] || row[:name].blank?++ case row[:mention_type]+ when Enums::MentionType::GROUP+ row[:target_id] = @names.group_id(row[:name])+ when Enums::MentionType::USER, nil+ row[:target_id] = @names.user_id(row[:name])+ end+ end+ end++ def resolve_hashtags(hashtag_rows)+ hashtag_rows.each do |row|+ next if row[:target_id] || row[:name].blank?++ resolved =+ case row[:hashtag_type]+ when Enums::HashtagType::CATEGORY+ @names.category_id(row[:name])&.then { |id| [id, Enums::HashtagType::CATEGORY] }+ when Enums::HashtagType::TAG+ @names.tag_id(row[:name])&.then { |id| [id, Enums::HashtagType::TAG] }+ else+ resolve_untyped_hashtag(row[:name])+ end+ next unless resolved++ row[:target_id], row[:hashtag_type] = resolved+ end+ end++ def resolve_untyped_hashtag(name)+ if (id = @names.category_id(name))+ [id, Enums::HashtagType::CATEGORY]+ elsif (id = @names.tag_id(name))+ [id, Enums::HashtagType::TAG]+ end+ end+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/placeholder_resolver.rb b/migrations/importer/lib/migrations/importer/placeholder_resolver.rbindex 31cd0c5b8fb..67c1e6992f0 100644--- a/migrations/importer/lib/migrations/importer/placeholder_resolver.rb+++ b/migrations/importer/lib/migrations/importer/placeholder_resolver.rb@@ -2,128 +2,92 @@module Migrationsmodule Importer- # The import-time counterpart of `EmbedBuffer`. It swaps the tokens left in an+ # The import-time counterpart of `EmbedBuffer`: swaps the tokens left in an# owner's markdown back to real Markdown, now that the `original_id ->- # discourse_id` maps exist. One resolver instance serves one owner kind- # (`owner_type`); currently the owner is always a post, later this will also- # cover user bios and other records.+ # discourse_id` maps exist. One resolver serves one owner kind.#- # It loads every linkage row for a batch of owners once (one query per kind),- # resolves the references the converter could only record by source coordinates- # or by name (a quoted post, a quoted user, a mentioned user or group), and then- # rewrites the bodies in memory. All of that runs in the load phase; no SQL runs- # while substituting. Turning a recorded name into a source id is delegated to- # {NameResolver}.+ # Linkage rows are loaded and their by-name and by-coordinate references+ # resolved ({NameResolver}) in the load phase; no SQL runs while+ # substituting.#- # ## The `maps` object- #- # Rendering needs the built import maps. They are passed in as one object so the- # resolver does no database work while substituting and stays easy to test.- # Every method must answer from memory — no SQL, no I/O: the render path calls- # the maps once per token, and the no-queries-while-substituting guarantee- # holds only if the maps keep it too. It must respond to:- #- # * `user(original_id)` => `{ username:, name: }` or `nil`- # * `group_name(original_id)` => `String` or `nil`- # * `post(original_id)` => `{ topic_id:, post_number: }` or `nil`- # * `topic_id(original_id)` => discourse topic id or `nil`- # * `upload_markdown(original_id)` => upload Markdown or `nil`- # * `poll_markdown(original_id)` => poll Markdown or `nil`- # * `event_markdown(original_id)` => event Markdown or `nil`- # * `category_slug_path(original_id)` => the destination category's slug path,- # `"slug"` or `"parent:child"`, or `nil`- # * `category_id(original_id)` => the destination category id or `nil`- # (an internal `/c/…` link needs the id,- # not just the slug path)- # * `tag_name(original_id)` => the destination tag's name or `nil`- # * `badge(original_id)` => `{ id:, slug: }` for the destination- # badge, or `nil`- # * `emoji_name(source_name)` => the destination custom emoji name (a- # conflict may rename it) or `nil`- # * `base_url` => the destination site's base URL- # * `here_mention` => the destination's `here_mention` site- # setting value (the name that acts as the- # "@here" mention)+ # The contract on a miss is verbatim restore: every row carries the exact+ # source substring it replaced, and an unresolved reference renders those+ # bytes unchanged. A hit rewrites as little as the row allows — links splice+ # only their recorded destination (and self-link label) spans, so titles,+ # angle brackets and padding survive. A canonical rebuild happens only when+ # something actually resolved, or for rows without a verbatim source.#- # ## Reporting+ # An unresolvable poll or event has no source value to fall back to and+ # disappears from the body; a token with no linkage row at all is stripped,+ # so no U+E000 character reaches the owner's markdown.#- # A missing lookup falls back to the source value. Uploads, polls and events- # have no source value to fall back to: when the map can't resolve one, the- # embed disappears. Each is sent to {#unresolved_embeds}.+ # ## The `maps` object#- # An internal link that can't be resolved does have a fallback (its source URL),- # but it is still reported: a stale internal link points at the wrong record- # rather than failing loudly, so operators need the report (see #render_link).+ # The built import maps arrive as one object whose methods must all answer+ # from memory: the render path calls them once per token, and that is what+ # keeps SQL out of substitution. Each answers nil when the destination has+ # no such record.#- # A token with no linkage row at all is an orphan — the token and its row were- # not written together upstream. It is stripped (so no U+E000 character reaches- # the owner's markdown) and sent to {#orphan_placeholders}. The unresolved reporting- # can't see this case, because there is no row behind it.+ # * `user(id)` => `{ username:, name: }`, `post(id)` => `{ topic_id:,+ # post_number: }`, `badge(id)` => `{ id:, slug: }`+ # * `upload(id)` => `{ short_url:, url: }`, the destination upload's+ # `upload://` spelling and its absolute URL+ # * `group_name(id)`, `tag_name(id)`, `tag_id(id)`, `topic_id(id)`, `category_id(id)`,+ # `category_slug_path(id)` (`"slug"` or `"parent:child"`)+ # * `upload_markdown(id)`, `poll_markdown(id)`, `event_markdown(id)`+ # * `emoji_name(name)`, keyed {NameNormalizer}-folded because a conflict+ # may have renamed the emoji+ # * `base_url` and `here_mention`, the destination's own valuesclass PlaceholderResolver- # An embed whose entity the maps couldn't resolve. Its token becomes an empty- # string, so this record is the only trace left. `owner_url` is the owning- # record's URL, or `nil` if it is not mapped.+ include TagLinkPath++ # An embed whose entity the maps couldn't resolve. A poll's or event's+ # token becomes an empty string, so this record is its only trace.UnresolvedEmbed = Data.define(:kind, :entity_id, :owner_id, :owner_url)- # A token with no linkage row. `kind` is parsed from the token, so a report can- # name what went missing. It matters most for quotes: stripping the opening-tag- # token leaves the `[/quote]` behind (see #render_quote).+ # `kind` is parsed from the token, so a report can name what went missing+ # — it matters most for quotes, where stripping the opening-tag token+ # leaves the `[/quote]` behind.OrphanPlaceholder = Data.define(:kind, :owner_id, :owner_url, :placeholder)- TABLES = {- quote: "embed_quotes",- link: "embed_links",- mention: "embed_mentions",- hashtag: "embed_hashtags",- emoji: "embed_emojis",- poll: "embed_polls",- event: "embed_events",- upload: "embed_uploads",- }.freeze- private_constant :TABLES-Enums = Migrations::Database::IntermediateDB::Enumsprivate_constant :Enums- # Where unresolved embeds and orphan tokens are reported. Each is whatever was- # passed to the constructor — anything that responds to `<<`. By default an Array- # you can read back after the run. For a large run, pass an object that writes- # straight to disk, so a systemic failure (say, every upload unresolved) does not- # keep one record per embed in memory.+ # Anything that responds to `<<`. For a large run, pass an object that+ # writes straight to disk, so a systemic failure does not keep one record+ # per embed in memory.attr_reader :unresolved_embeds, :orphan_placeholders- # @param owner_type [Integer] the owner kind this resolver serves, an- # `Enums::EmbedOwner` value (e.g. `EmbedOwner::POST`).# @param maps see the class description for the methods it must answer.- # @param unresolved_embeds [#<<] collects {UnresolvedEmbed}s.- # @param orphan_placeholders [#<<] collects {OrphanPlaceholder}s.+ # @param trusted_upload_hosts [Enumerable<String>] external hosts whose+ # upload URLs belong to the source site, such as its CDN. Only their+ # recorded upload ids may use the imported upload map.def initialize(intermediate_db,maps,owner_type:,+ trusted_upload_hosts: [],unresolved_embeds: [],orphan_placeholders: [])- @intermediate_db = intermediate_db@maps = maps@owner_type = owner_type@unresolved_embeds = unresolved_embeds@orphan_placeholders = orphan_placeholders- @names = NameResolver.new(intermediate_db)+ @trusted_upload_hosts = trusted_upload_hosts.map { |host| host.to_s.downcase }.to_set+ @linkages = PlaceholderLinkages.new(intermediate_db)end- # @param items [Array<Hash>] each with `:id` (the owner's original_id) and `:raw`.+ # @param items [Array<Hash>] each with `:id` (the owner's original_id) and+ # `:raw`.# @return [Hash{Object => String}] owner original_id => resolved raw.def resolve_all(items)- # An owner with no token in its raw has no embeds (the token and the row are- # written together), so there's nothing to load for it. Most bodies are+ # An owner with no token in its raw has no embeds, and most bodies are# plain text, so this skips the linkage queries for the bulk of a batch.- # `String#include?` of the one-char delimiter is a `memchr`, far cheaper- # than querying every linkage table per owner.with_embeds =items.select { |item| item[:raw]&.include?(Migrations::Placeholder::DELIMITER) }- linkages = load_linkages(with_embeds.map { |item| item[:id] })- resolve_linkage_ids(linkages)+ linkages =+ @linkages.load_and_resolve(with_embeds.map { |item| item[:id] }, owner_type: @owner_type)items.each_with_object({}) do |item, result|body = substitute(item[:raw], linkages[item[:id]])@@ -133,15 +97,9 @@ module Migrationsprivate- # Rewrites the tokens in one body in a single `gsub` pass — one scan of the body- # instead of one `gsub` per embed.- #- # The block form is required. With a string replacement, `gsub` reads `\1`,- # `\0` etc. in the rendered Markdown as backreferences and drops backslashes,- # which corrupts user content. The block copies the text unchanged.- #- # `linkage_rows` is a list of `[kind, row]` pairs (see #load_linkages): each- # row needs its kind, and the kind is not stored on the row.+ # The block form is required: with a string replacement, `gsub` reads+ # `\1`, `\0` etc. in the rendered Markdown as backreferences and drops+ # backslashes, which corrupts user content.def substitute(raw, linkage_rows)return raw if raw.nil? || linkage_rows.blank?@@ -154,8 +112,6 @@ module Migrationsendend- # Strips and records any token still here after substitution — it had no- # linkage row. Usually there's none, so this is just one cheap check.def strip_orphans(body, owner_id)return body unless body && Migrations::Placeholder.include?(body)@@ -174,190 +130,6 @@ module Migrationsbody.gsub(Migrations::Placeholder::PATTERN, "")end- # One query per kind, grouped by owner id. The only place that reads the database.- #- # Each query binds one variable per owner id, so a batch has to stay well- # under SQLite's variable limit (999 on old builds, 32766 on current ones).- # That limit is the real ceiling on batch size; the caller picks the size.- def load_linkages(owner_ids)- buckets = {}- return buckets if owner_ids.empty?-- bind_params = (["?"] * owner_ids.size).join(", ")-- TABLES.each do |kind, table|- sql = "SELECT * FROM #{table} WHERE owner_type = ? AND owner_id IN (#{bind_params})"- @intermediate_db.query(sql, @owner_type, *owner_ids) do |row|- (buckets[row[:owner_id]] ||= []) << [kind, row]- end- end-- buckets- end-- # Fills in the ids that can only be resolved once the whole IntermediateDB is- # loaded: a quoted post named by source coordinates, and a quoted user or- # mentioned user/group named by name. Runs once per batch in the load phase, so- # no query runs while substituting. Rows are mutated in place; from here on the- # render path treats them as if the converter had known the ids all along.- def resolve_linkage_ids(linkages)- rows_by_kind = Hash.new { |hash, kind| hash[kind] = [] }- linkages.each_value { |pairs| pairs.each { |kind, row| rows_by_kind[kind] << row } }-- # Turn the source coordinates (topic id + post number) a quote carries into- # the quoted post's source original_id. A miss leaves it nil, so the username- # fallback still applies.- fill_post_ids(- rows_by_kind[:quote],- id: :quoted_post_id,- topic: :quoted_topic_id,- number: :quoted_post_number,- )- resolve_quoted_usernames(rows_by_kind[:quote])- resolve_mention_names(rows_by_kind[:mention])- resolve_hashtags(rows_by_kind[:hashtag])- resolve_links(rows_by_kind[:link])- end-- # Fills the nil `id` column for rows that carry source `(topic, number)`- # coordinates, via one batched lookup. A pair with no matching post leaves the- # id nil, so callers keep their fallback.- def fill_post_ids(rows, id:, topic:, number:)- pending = rows.select { |row| row[id].nil? && row[topic] && row[number] }- return if pending.empty?-- post_ids = post_ids_for_coordinates(pending.map { |row| [row[topic], row[number]] })- pending.each { |row| row[id] = post_ids[[row[topic], row[number]]] }- end-- # Batch-resolves source `(topic_id, post_number)` coordinates to source post- # original_ids. Shared by the quote and internal-link passes — both carry a post- # by its source coordinates, and one query answers the whole batch: SQLite row- # values match the pairs against the `(topic_id, post_number)` index. A pair with- # no matching post is simply absent from the result, so each caller keeps its own- # fallback.- #- # Two binds per pair, so the same SQLite variable-limit ceiling applies (see- # #load_linkages).- def post_ids_for_coordinates(coordinates)- coordinates = coordinates.uniq- return {} if coordinates.empty?-- values = (["(?, ?)"] * coordinates.size).join(", ")- sql = <<~SQL- SELECT original_id, topic_id, post_number- FROM posts- WHERE (topic_id, post_number) IN (VALUES #{values})- SQL-- post_ids = {}- @intermediate_db.query(sql, *coordinates.flatten) do |row|- post_ids[[row[:topic_id], row[:post_number]]] = row[:original_id]- end- post_ids- end-- # Fills an internal link's `target_id` from what the URL carried but the- # converter couldn't resolve: a post addressed by coordinates, or an entity- # addressed by name. Both run in the load phase, so no query runs while- # substituting. On a miss nothing is mutated; render_link falls back to the- # source URL (and reports it).- def resolve_links(link_rows)- # A `/t/slug/<topic>/<post>` link records the post by its source coordinates,- # exactly like a quote; resolve it through the same batch lookup.- fill_post_ids(- link_rows,- id: :target_id,- topic: :target_topic_id,- number: :target_post_number,- )- resolve_link_names(link_rows)- end-- # A link that named its target (a username, group/tag name, or category slug- # path) resolves against the same {NameResolver} the mention and hashtag- # passes use. A category name goes through `category_id`, so a `parent:child`- # path and a bare slug resolve the same way a hashtag does.- def resolve_link_names(link_rows)- link_rows.each do |row|- next if row[:target_id] || row[:target_name].blank?-- row[:target_id] = lookup_target_id(row[:target_type], row[:target_name])- end- end-- def lookup_target_id(target_type, name)- case target_type- when Enums::LinkTarget::USER- @names.user_id(name)- when Enums::LinkTarget::GROUP- @names.group_id(name)- when Enums::LinkTarget::TAG- @names.tag_id(name)- when Enums::LinkTarget::CATEGORY- @names.category_id(name)- end- end-- # Fills a quoted user's id from the recorded name, for rows that carry a name- # but no id.- def resolve_quoted_usernames(quote_rows)- quote_rows.each do |row|- next if row[:quoted_user_id] || row[:quoted_username].blank?- row[:quoted_user_id] = @names.user_id(row[:quoted_username])- end- end-- # Fills a mention's target id from the recorded name, for rows that carry a- # name but no id. `here`/`all` mentions name no entity.- def resolve_mention_names(mention_rows)- mention_rows.each do |row|- next if row[:target_id] || row[:name].blank?-- case row[:mention_type]- when Enums::MentionType::GROUP- row[:target_id] = @names.group_id(row[:name])- when Enums::MentionType::USER, nil- row[:target_id] = @names.user_id(row[:name])- end- end- end-- # Resolves each hashtag's name to a source category or tag original_id and- # pins its type. A row the source forced with `::tag`/`::category` is resolved- # against that type only; an untyped row tries a category first, then a tag,- # the way Discourse cooks a bare `#slug`. On a miss nothing is mutated:- # `target_id` stays nil and a forced `hashtag_type` survives, so rendering can- # rebuild the source text faithfully.- def resolve_hashtags(hashtag_rows)- hashtag_rows.each do |row|- next if row[:target_id] || row[:name].blank?-- resolved =- case row[:hashtag_type]- when Enums::HashtagType::CATEGORY- @names.category_id(row[:name])&.then { |id| [id, Enums::HashtagType::CATEGORY] }- when Enums::HashtagType::TAG- @names.tag_id(row[:name])&.then { |id| [id, Enums::HashtagType::TAG] }- else- resolve_untyped_hashtag(row[:name])- end- next unless resolved-- row[:target_id], row[:hashtag_type] = resolved- end- end-- # A bare `#name` is a category first, a tag second — the order Discourse falls- # back through when the source didn't pin the type.- def resolve_untyped_hashtag(name)- if (id = @names.category_id(name))- [id, Enums::HashtagType::CATEGORY]- elsif (id = @names.tag_id(name))- [id, Enums::HashtagType::TAG]- end- end-def render(kind, row)case kindwhen :quote@@ -375,8 +147,6 @@ module Migrationsendend- # Embeds whose Markdown comes from the maps. No map hit means no fallback, so- # the embed drops out (empty string) and is recorded.def render_entity(kind, row)entity_id, markdown =case kind@@ -385,12 +155,12 @@ module Migrationswhen :event[row[:event_id], @maps.event_markdown(row[:event_id])]when :upload- [row[:upload_id], @maps.upload_markdown(row[:upload_id])]+ [row[:upload_id], resolve_upload(row)]endreturn markdown if markdown.present?- # Always report: the report is the only signal this entity needs attention.+ # The report is the only signal this entity needs attention.@unresolved_embeds << UnresolvedEmbed.new(kind:,entity_id:,@@ -398,68 +168,176 @@ module Migrationsowner_url: owner_url_for(row[:owner_id]),)- # An upload referenced by a full URL carries the verbatim source snippet;- # put it back rather than dropping it, so a hotlink to another forum's upload- # (which maps to nothing here) survives. Polls, events and `upload://`- # uploads have no fallback and drop out.+ # An upload row carries the verbatim source snippet, so a hotlink to+ # another forum's upload survives. Polls and events have no fallback.kind == :upload ? row[:original_markdown].to_s : ""end- # Builds the opening `[quote="…"]` tag only; the quoted text and `[/quote]` are- # plain text in the raw (see EmbedBuffer#quote). So an orphaned quote token,- # once stripped, leaves its `[/quote]` behind.+ # A foreign 40-hex basename could collide with a source upload sha1 and+ # rewrite another site's file, so an unrecognized host's row is mapped+ # only when the operator trusts that host as source upload storage.+ def resolve_upload(row)+ external_host = row[:external_host]+ return nil if external_host && !@trusted_upload_hosts.include?(external_host.downcase)++ render_upload(row)+ end++ # A snippet that is nothing but a destination — a reference definition's+ # URL, or a full URL core linkified on its own — has to come back as a URL+ # in the source's own spelling: image markdown would break the definition+ # and turn the prose link into an image. Only a definition can hold a bare+ # `upload://`, because core doesn't tokenize one in prose, which is also+ # why a full URL can't be answered with the short form.+ def render_upload(row)+ snippet = row[:original_markdown].to_s+ if snippet.empty? || snippet.start_with?("![", "[")+ return @maps.upload_markdown(row[:upload_id])+ end++ spelling = snippet.start_with?("upload://") ? :short_url : :url+ @maps.upload(row[:upload_id])&.dig(spelling)+ end++ # Builds the opening `[quote="…"]` tag only (see EmbedBuffer#quote). A+ # canonical rebuild is only warranted when something actually resolved; a+ # full miss restores the verbatim source tag, which may carry syntax the+ # reference columns don't model (casing, spacing, parameters core+ # ignores). Recorded post coordinates that don't resolve take that+ # fallback too, however well the quoted user resolved: rendering the user+ # alone would turn a quote of one specific post into a bare attribution.def render_quote(row)user = row[:quoted_user_id] ? @maps.user(row[:quoted_user_id]) : nilusername = user&.dig(:username) || row[:quoted_username]name = user&.dig(:name) || row[:quoted_name]- if row[:quoted_post_id] && (post = @maps.post(row[:quoted_post_id]))- topic_id = post[:topic_id]- post_number = post[:post_number]- end+ post = row[:quoted_post_id] ? @maps.post(row[:quoted_post_id]) : nil+ topic_id = post&.dig(:topic_id)+ post_number = post&.dig(:post_number)++ lost_coordinates = quoted_post_recorded?(row) && !(topic_id && post_number)+ report_unresolved_quote(row) if lost_coordinates- return "[quote]" if username.blank? && name.blank?+ if row[:original_markdown].present? && (lost_coordinates || (user.nil? && post.nil?))+ return row[:original_markdown]+ end- parts = []- parts << (name.presence || username)+ parts = [name.presence || username.presence]parts << "post:#{post_number}" if post_number.present?parts << "topic:#{topic_id}" if topic_id.present?parts << "username:#{username}" if username.present? && name.present? && name != username+ return "[quote]" if parts.compact.empty?++ # Core reads the first segment as the username, so it would attribute+ # `[quote="post:4, topic:9"]` to a user named "post:4"."[quote=\"#{parts.join(", ")}\"]"end- # An external link (no `target_type`) passes through as-is. An internal link is- # rebuilt through the resolved `target_id` and the destination maps, honoring any- # rename or renumber; whatever trailed the matched route (`target_suffix`) is- # reattached verbatim. A bare URL stays bare so oneboxes keep working; a link- # with text keeps its `[text](url)` shape.- #- # An internal link that can't be resolved (nil `target_id` after normalization,- # or a maps miss here) falls back to the source URL AND reports it. This diverges- # from the mention/hashtag no-report convention on purpose: on a merge into an- # existing site a stale `/t/slug/123` doesn't 404, it silently points at the- # WRONG topic, so operators need the audit trail.+ # Any of the three columns counts: a pair the source posts don't have and+ # a lone coordinate are both unresolvable, and both would vanish.+ def quoted_post_recorded?(row)+ row[:quoted_post_id].present? || row[:quoted_topic_id].present? ||+ row[:quoted_post_number].present?+ end++ def report_unresolved_quote(row)+ entity_id = row[:quoted_post_id] || quoted_coordinates(row)++ @unresolved_embeds << UnresolvedEmbed.new(+ kind: :quote,+ entity_id:,+ owner_id: row[:owner_id],+ owner_url: owner_url_for(row[:owner_id]),+ )+ end++ def quoted_coordinates(row)+ [+ ("topic:#{row[:quoted_topic_id]}" if row[:quoted_topic_id].present?),+ ("post:#{row[:quoted_post_number]}" if row[:quoted_post_number].present?),+ ].compact.join(", ")+ end++ # An unresolvable internal link falls back to the source URL AND reports+ # it. This diverges from the mention/hashtag no-report convention on+ # purpose: on a merge into an existing site a stale `/t/slug/123` doesn't+ # 404, it silently points at the WRONG topic. A `SITE` link only has its+ # origin rewritten, so it always resolves and is never reported.def render_link(row)return render_link_markup(row, row[:url]) unless row[:target_type]+ return render_site_link(row) if row[:target_type] == Enums::LinkTarget::SITE- url = row[:target_id] && rebuild_internal_link(row)+ url =+ if multi_tag_link?(row)+ rebuild_multi_tag_link(row)+ else+ row[:target_id] && rebuild_internal_link(row)+ endreturn render_link_markup(row, url) if urlreport_unresolved_link(row)render_link_markup(row, row[:url])end- # `presence`: a converter may have recorded an empty text; treat it as a- # bare URL rather than rendering `[](url)`.+ def render_site_link(row)+ render_link_markup(row, "#{@maps.base_url}#{row[:target_suffix]}")+ end++ # Rewrites only the destination inside the verbatim source construct, so+ # titles, angle brackets and padding survive and an unchanged URL+ # round-trips byte-exact. Every spelling of the source URL inside the+ # construct is a destination spelling (a `[URL](URL)` self-link shows it+ # twice), so all are rewritten.def render_link_markup(row, url)+ original = row[:original_markdown]+ if original.present?+ return original if url == row[:url]++ spans = destination_spans(row, original)+ return splice_url_spans(original, spans, url) if spans+ end++ # An empty recorded text renders as a bare URL rather than `[](url)`.+ # Also the path for a row with a snippet but no destination span: a+ # value search would rewrite the URL wherever else the author typed it.text = row[:text].presencetext ? "[#{text}](#{url})" : url.to_send- # The destination URL for a resolved internal link, or nil on a maps miss (an- # entity the destination doesn't have). The suffix is appended by the caller's- # success path, so a miss can report cleanly.+ # The destination's recorded byte span(s) inside the verbatim snippet, or+ # nil when the row carries none or a span is wrong. The bytes at the+ # offset must equal the row's URL: an offset that is in bounds but points+ # elsewhere would splice the destination into arbitrary syntax.+ def destination_spans(row, original)+ offset = row[:url_offset]+ return nil if offset.nil? || row[:url].blank?++ url = row[:url]+ length = url.bytesize+ return nil unless span_holds_url?(original, offset, url)++ spans = [offset]+ label = row[:label_url_offset]+ spans << label if label && label != offset && span_holds_url?(original, label, url)+ spans.sort.map { |span_offset| [span_offset, length] }+ end++ # A negative offset would read from the end, and rows come from a+ # database.+ def span_holds_url?(original, offset, url)+ offset >= 0 && original.byteslice(offset, url.bytesize) == url+ end++ def splice_url_spans(original, spans, url)+ result = original.dup+ spans.reverse_each { |offset, length| result.bytesplice(offset, length, url) }+ result+ end++ # The destination URL for a resolved internal link, or nil on a maps miss.+ # The suffix is appended by the caller's success path, so a miss can+ # report cleanly.def rebuild_internal_link(row)base = internal_link_base(row)base && "#{base}#{row[:target_suffix]}"@@ -480,7 +358,9 @@ module Migrationswhen Enums::LinkTarget::GROUP(name = @maps.group_name(target_id)) && "#{@maps.base_url}/g/#{name}"when Enums::LinkTarget::TAG- (name = @maps.tag_name(target_id)) && "#{@maps.base_url}/tag/#{name}"+ name = @maps.tag_name(target_id)+ id = @maps.tag_id(target_id)+ name && id && "#{@maps.base_url}/tag/#{name}/#{id}"when Enums::LinkTarget::CATEGORYcategory_link_url(target_id)when Enums::LinkTarget::BADGE@@ -488,12 +368,43 @@ module Migrationsendend- # `/c/<slug path>/<id>`, with the slug path's `:` separators turned back into- # `/`. Both the id and the path come from the destination category.+ # `/c/<slug path>/<id>`, with the slug path's `:` separators turned back+ # into `/`.def category_link_url(target_id)+ parts = category_route_parts(target_id)+ parts && "#{@maps.base_url}/c/#{parts}"+ end++ # `<slug path>/<id>` for the destination category — the shared middle of+ # the `/c/…` and `/tags/c/…` routes. Nil when the category isn't mapped.+ def category_route_parts(target_id)+ return nil unless target_id+new_id = @maps.category_id(target_id)path = @maps.category_slug_path(target_id)- new_id && path && "#{@maps.base_url}/c/#{path.tr(":", "/")}/#{new_id}"+ new_id && path && "#{path.tr(":", "/")}/#{new_id}"+ end++ # The destination URL for a multi-tag route. Nil when any coordinate+ # didn't map, because a route naming several records is only rebuilt+ # whole.+ def rebuild_multi_tag_link(row)+ tag_ids = row[:resolved_tag_ids]+ return nil unless tag_ids++ names = tag_ids.map { |id| @maps.tag_name(id) }+ return nil unless names.all?++ base =+ case row[:target_type]+ when Enums::LinkTarget::TAG_INTERSECTION+ "#{@maps.base_url}/tags/intersection/#{names.join("/")}"+ when Enums::LinkTarget::CATEGORY_TAG+ parts = category_route_parts(row[:target_id])+ tag_path = [tag_path_filter(row), names.first].compact.join("/")+ parts && "#{@maps.base_url}/tags/c/#{parts}/#{tag_path}"+ end+ base && "#{base}#{row[:target_suffix]}"enddef badge_link_url(target_id)@@ -502,8 +413,8 @@ module Migrationsenddef report_unresolved_link(row)- # A coordinate-form post link has neither a target id nor a name; the- # original URL is the most useful identifier a report can carry for it.+ # A coordinate-form post link has neither a target id nor a name, so the+ # original URL is the most useful thing a report can name it by.@unresolved_embeds << UnresolvedEmbed.new(kind: :link,entity_id: row[:target_id] || row[:target_name] || row[:url],@@ -516,37 +427,40 @@ module Migrationsname =case row[:mention_type]when Enums::MentionType::HERE- # The destination decides which name acts as the here-mention.- @maps.here_mention.presence || "here"+ # Here-matching is case-insensitive at cook time, so the author's+ # own spelling works as-is — only a different name is a remap.+ here = @maps.here_mention.presence+ if here && (row[:name].blank? || !here.casecmp?(row[:name]))+ here+ else+ row[:name].presence || "here"+ endwhen Enums::MentionType::ALL- "all"+ row[:name].presence || "all"when Enums::MentionType::GROUP@maps.group_name(row[:target_id]) || row[:name]else # USER, or an unspecified (nil) mention@maps.user(row[:target_id])&.dig(:username) || row[:name]end- # The token spans exactly the original `@name`, so the surrounding text is- # already intact — rendering verbatim keeps the source spacing.+ # The token spans exactly the original `@name`, so the surrounding text+ # is already intact.return "@#{name}" if name.present?- # Nearly unreachable: the converter always records a name. But an embed may- # only vanish with a report, so record one before dropping it.+ # Nearly unreachable, but an embed may only vanish with a report.@unresolved_embeds << UnresolvedEmbed.new(kind: :mention,entity_id: row[:target_id] || row[:name],owner_id: row[:owner_id],owner_url: owner_url_for(row[:owner_id]),)- ""+ row[:original_markdown].to_send- # A resolved category renders as `#<slug path>`, honoring any rename or merge- # the destination applied. A resolved tag renders as `#<name>::tag` always: a- # bare `#name` resolves category-first at import, so a destination category- # sharing the slug would otherwise hijack an unsuffixed tag. A map miss or an- # unresolved row rebuilds the source text, so the original `#name` survives.- # No unresolved report — the source value is the fallback, as with mentions.+ # A resolved tag renders as `#<name>::tag` always: a bare `#name` resolves+ # category-first at import, so a destination category sharing the slug+ # would otherwise hijack an unsuffixed tag. No unresolved report — the+ # source value is the fallback, as with mentions.def render_hashtag(row)if row[:target_id]case row[:hashtag_type]@@ -559,12 +473,14 @@ module Migrationsendend- rebuild_hashtag(row)+ # A bare `#name` must stay bare even when the import resolved a type it+ # then couldn't render: the mutated `hashtag_type` must not leak a+ # suffix the source never had.+ row[:original_markdown].presence || rebuild_hashtag(row)end- # Rebuilds the source `#name`, re-adding the `::tag`/`::category` suffix the- # `hashtag_type` implies (present only when the source forced it or the import- # resolved a type it then couldn't render).+ # Rebuilds the source `#name` for a row without a verbatim source, adding+ # back the suffix the `hashtag_type` implies.def rebuild_hashtag(row)suffix =case row[:hashtag_type]@@ -579,12 +495,11 @@ module Migrations"##{row[:name]}#{suffix}"end- # A custom emoji renders as `:<name>:`, remapped through the emoji-name map in- # case a conflict renamed it. A map miss puts the source name back verbatim; no- # unresolved report — the source value is the fallback, as with mentions and- # hashtags.+ # The lookup folds the way {NameResolver}'s name lookups do, because the+ # row keeps the author's spelling (`:MYEMOJI:`) while core lowercases a+ # shortcode first. No unresolved report — the source name is the fallback.def render_emoji(row)- name = @maps.emoji_name(row[:name]) || row[:name]+ name = @maps.emoji_name(Migrations::NameNormalizer.normalize(row[:name])) || row[:name]":#{name}:"end@@ -596,8 +511,7 @@ module Migrations"#{@maps.base_url}/t/#{post[:topic_id]}/#{post[:post_number]}"end- # The URL of the record a token sits in, for reporting. `nil` if that record- # is not mapped (it normally is by the time we substitute).+ # Nil when the owning record is not mapped.def owner_url_for(owner_id)case @owner_typewhen Enums::EmbedOwner::POSTdiff --git a/migrations/importer/lib/migrations/importer/tag_link_path.rb b/migrations/importer/lib/migrations/importer/tag_link_path.rbnew file mode 100644index 00000000000..c4e53e332ad--- /dev/null+++ b/migrations/importer/lib/migrations/importer/tag_link_path.rb@@ -0,0 +1,37 @@+# frozen_string_literal: true++module Migrations+ module Importer+ # ID resolution and URL rendering must agree on which path segments are+ # tag names and which are category filters.+ module TagLinkPath+ Enums = Migrations::Database::IntermediateDB::Enums+ private_constant :Enums++ TAG_SUBCATEGORY_FILTERS = %w[none all]+ private_constant :TAG_SUBCATEGORY_FILTERS++ private++ def multi_tag_link?(row)+ row[:target_type] == Enums::LinkTarget::CATEGORY_TAG ||+ row[:target_type] == Enums::LinkTarget::TAG_INTERSECTION+ end++ def tag_path_filter(row)+ return nil unless row[:target_type] == Enums::LinkTarget::CATEGORY_TAG++ segments = row[:target_tag_path].to_s.split("/")+ return nil unless segments.size > 1++ TAG_SUBCATEGORY_FILTERS.include?(segments.first) ? segments.first : nil+ end++ def tag_path_tags(row)+ segments = row[:target_tag_path].to_s.split("/")+ segments.shift if tag_path_filter(row)+ segments+ end+ end+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_linkages_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_linkages_spec.rbnew file mode 100644index 00000000000..fb4fdf76697--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_linkages_spec.rb@@ -0,0 +1,85 @@+# frozen_string_literal: true++require "tmpdir"++RSpec.describe Migrations::Importer::PlaceholderLinkages do+ let(:embed_owner) { Migrations::Database::IntermediateDB::Enums::EmbedOwner }+ let(:link_target) { Migrations::Database::IntermediateDB::Enums::LinkTarget }++ it "resolves owner ids, coordinate pairs, and slugs within the configured bind limit" do+ 4.times do |index|+ Migrations::Database::IntermediateDB::Topic.create(+ original_id: index + 10,+ title: "Topic #{index}",+ slug: "topic-#{index}",+ )+ Migrations::Database::IntermediateDB::EmbedLink.create(+ owner_type: embed_owner::POST,+ owner_id: index + 1,+ placeholder: "link-#{index}",+ target_type: link_target::TOPIC,+ target_name: "topic-#{index}",+ )+ end++ intermediate_db.execute(<<~SQL)+ CREATE TABLE posts (+ original_id NUMERIC NOT NULL,+ topic_id NUMERIC NOT NULL,+ post_number INTEGER NOT NULL+ )+ SQL+ [[100, 20, 2], [101, 21, 3]].each do |original_id, topic_id, post_number|+ intermediate_db.execute(+ "INSERT INTO posts (original_id, topic_id, post_number) VALUES (?, ?, ?)",+ original_id,+ topic_id,+ post_number,+ )+ end+ [[5, 20, 2], [6, 21, 3]].each do |owner_id, topic_id, post_number|+ Migrations::Database::IntermediateDB::EmbedQuote.create(+ owner_type: embed_owner::POST,+ owner_id:,+ placeholder: "quote-#{owner_id}",+ quoted_topic_id: topic_id,+ quoted_post_number: post_number,+ )+ end++ real_db = intermediate_db+ limited_db = Object.new+ limited_db.define_singleton_method(:query) do |sql, *parameters, &block|+ raise "query exceeded bind limit" if parameters.size > 3++ real_db.query(sql, *parameters, &block)+ end+ linkages = described_class.new(limited_db, bind_limit: 3)++ resolved = linkages.load_and_resolve((1..6).to_a, owner_type: embed_owner::POST)++ links = resolved.values.flatten(1).filter_map { |kind, row| row if kind == :link }+ quotes = resolved.values.flatten(1).filter_map { |kind, row| row if kind == :quote }+ expect(links.map { |row| row[:target_id] }).to contain_exactly(10, 11, 12, 13)+ expect(quotes.map { |row| row[:quoted_post_id] }).to contain_exactly(100, 101)+ end++ around do |example|+ Dir.mktmpdir do |dir|+ db_path = File.join(dir, "intermediate.db")+ Migrations::Database.migrate(+ db_path,+ migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,+ )+ @intermediate_db = Migrations::Database.connect(db_path)+ Migrations::Database::IntermediateDB.setup(@intermediate_db)+ example.run+ ensure+ Migrations::Database::IntermediateDB.setup(nil)+ end+ end++ def intermediate_db+ @intermediate_db+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rbindex 552a99229ab..f5941dd780e 100644--- a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb@@ -4,6 +4,31 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doinclude_context "with placeholder resolver"describe "resolving recorded names to source original_ids" do+ it "resolves distinct sigma usernames to their own users" do+ create_user(10, "κοσμος")+ create_user(20, "κοσμοσ")+ maps =+ FakePlaceholderMaps.new(user: { 10 => { username: "first" }, 20 => { username: "second" } })+ tokens =+ %w[κοσμος κοσμοσ ΚΟΣΜΟΣ].map do |name|+ create_embed(:mention, mention_type: mention_type::USER, name:)+ end+ quote = create_embed(:quote, quoted_username: "κοσμος")+ link = create_embed(:link, target_type: link_target::USER, target_name: "κοσμοσ")++ expect(resolve([*tokens, quote, link].join(" "), maps:)).to eq(+ '@first @second @second [quote="first"] https://dest.example.com/u/second',+ )+ end++ it "leaves a mention unresolved when only the other sigma username exists" do+ create_user(20, "κοσμοσ")+ maps = FakePlaceholderMaps.new(user: { 20 => { username: "second" } })+ token = create_embed(:mention, mention_type: mention_type::USER, name: "κοσμος")++ expect(resolve(token, maps:)).to eq("@κοσμος")+ end+it "maps a quoted username to the user's original_id, honoring an import-time rename" doMigrations::Database::IntermediateDB::User.create(original_id: 5,@@ -94,33 +119,9 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doenddescribe "hashtag resolution" do- def create_hashtag(placeholder_token, **attrs)- Migrations::Database::IntermediateDB::EmbedHashtag.create(- owner_type: embed_owner::POST,- owner_id: 1,- placeholder: placeholder_token,- **attrs,- )- end-- def create_category(original_id, slug, parent_category_id: nil)- Migrations::Database::IntermediateDB::Category.create(- original_id:,- name: slug,- slug:,- parent_category_id:,- user_id: 1,- )- end-- def create_tag(original_id, name)- Migrations::Database::IntermediateDB::Tag.create(original_id:, name:, slug: name)- end-it "resolves a bare hashtag to a category, honoring an import-time slug rename" docreate_category(10, "support")- hashtag = placeholder.mint(:hashtag)- create_hashtag(hashtag, name: "support")+ hashtag = create_embed(:hashtag, name: "support")maps = FakePlaceholderMaps.new(category_slug_path: { 10 => "help" })resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)@@ -132,8 +133,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doit "resolves a parent:child hashtag by its full slug path" docreate_category(10, "support")create_category(11, "billing", parent_category_id: 10)- hashtag = placeholder.mint(:hashtag)- create_hashtag(hashtag, name: "support:billing")+ hashtag = create_embed(:hashtag, name: "support:billing")maps = FakePlaceholderMaps.new(category_slug_path: { 11 => "support:billing" })resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)@@ -146,8 +146,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver docreate_category(10, "news")create_category(20, "parent")create_category(21, "news", parent_category_id: 20)- hashtag = placeholder.mint(:hashtag)- create_hashtag(hashtag, name: "news")+ hashtag = create_embed(:hashtag, name: "news")maps = FakePlaceholderMaps.new(category_slug_path: { 10 => "news", 21 => "parent:news" })resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)@@ -158,8 +157,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doit "resolves a bare hashtag to a tag when no category matches, always suffixing ::tag" docreate_tag(30, "release")- hashtag = placeholder.mint(:hashtag)- create_hashtag(hashtag, name: "release")+ hashtag = create_embed(:hashtag, name: "release")maps = FakePlaceholderMaps.new(tag_name: { 30 => "shipped" })resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)@@ -172,8 +170,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver docreate_tag(30, "release")create_tag(31, "ship")Migrations::Database::IntermediateDB::TagSynonym.create(synonym_tag_id: 31, target_tag_id: 30)- hashtag = placeholder.mint(:hashtag)- create_hashtag(hashtag, name: "ship")+ hashtag = create_embed(:hashtag, name: "ship")maps = FakePlaceholderMaps.new(tag_name: { 30 => "release" })resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)@@ -185,8 +182,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doit "skips category precedence when the source forced the tag type" docreate_category(10, "release")create_tag(30, "release")- hashtag = placeholder.mint(:hashtag)- create_hashtag(hashtag, name: "release", hashtag_type: hashtag_type::TAG)+ hashtag = create_embed(:hashtag, name: "release", hashtag_type: hashtag_type::TAG)maps =FakePlaceholderMaps.new(category_slug_path: {@@ -205,10 +201,8 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doendit "rebuilds the source text for an unresolved hashtag, keeping a source-forced suffix" do- forced = placeholder.mint(:hashtag)- bare = placeholder.mint(:hashtag)- create_hashtag(forced, name: "ghost", hashtag_type: hashtag_type::CATEGORY)- create_hashtag(bare, name: "missing")+ forced = create_embed(:hashtag, name: "ghost", hashtag_type: hashtag_type::CATEGORY)+ bare = create_embed(:hashtag, name: "missing")resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)resolved = resolver.resolve_all([{ id: 1, raw: "a #{forced} b #{bare} c" }])@@ -219,8 +213,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doit "rebuilds the source text when the name resolved but the destination dropped it" docreate_category(10, "support")- hashtag = placeholder.mint(:hashtag)- create_hashtag(hashtag, name: "support")+ hashtag = create_embed(:hashtag, name: "support")# Category resolves against the IDB, but the maps have no destination slug for it.resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rbindex 8a4702008f3..f8ee969f199 100644--- a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb@@ -40,59 +40,64 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doexpect(resolved[1]).to eq("x https://dest.example.com/t/42/3 y")end- it "keeps the source URL for a link without a target" do+ it "rewrites a SITE link's origin, keeping its path, query and fragment" dolink = placeholder.mint(:link)Migrations::Database::IntermediateDB::EmbedLink.create(owner_type: embed_owner::POST,owner_id: 1,placeholder: link,- url: "https://elsewhere.example.com/page",+ url: "https://old.example.com/faq?x=1#y",+ target_type: link_target::SITE,+ target_suffix: "/faq?x=1#y",)+ maps = FakePlaceholderMaps.new+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])- expect(resolved[1]).to eq("x https://elsewhere.example.com/page y")+ expect(resolved[1]).to eq("x https://dest.example.com/faq?x=1#y y")+ expect(resolver.unresolved_embeds).to be_emptyend- end- describe "internal link resolution" do- def create_link(placeholder_token, **attrs)+ it "writes a SITE suffix back in the author's spelling" do+ # The suffix is stored as the author wrote it — a quote stays a quote,+ # never the percent-encoding the parser's normalized href used.+ link = placeholder.mint(:link)Migrations::Database::IntermediateDB::EmbedLink.create(owner_type: embed_owner::POST,owner_id: 1,- placeholder: placeholder_token,- **attrs,+ placeholder: link,+ url: %{https://old.example.com/new?public_key="abc"},+ target_type: link_target::SITE,+ target_suffix: %{/new?public_key="abc"},)- end+ maps = FakePlaceholderMaps.new+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)- def create_user(original_id, username)- Migrations::Database::IntermediateDB::User.create(- original_id:,- username:,- created_at: Time.now,- trust_level: 0,- )+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])++ expect(resolved[1]).to eq(%{x https://dest.example.com/new?public_key="abc" y})+ expect(resolver.unresolved_embeds).to be_emptyend- def create_category(original_id, slug, parent_category_id: nil)- Migrations::Database::IntermediateDB::Category.create(- original_id:,- name: slug,- slug:,- parent_category_id:,- user_id: 1,+ it "keeps the source URL for a link without a target" do+ link = placeholder.mint(:link)+ Migrations::Database::IntermediateDB::EmbedLink.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: link,+ url: "https://elsewhere.example.com/page",)- end- def create_tag(original_id, name)- Migrations::Database::IntermediateDB::Tag.create(original_id:, name:, slug: name)+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])++ expect(resolved[1]).to eq("x https://elsewhere.example.com/page y")end+ end+ describe "internal link resolution" dodef render(attrs, maps:)- link = placeholder.mint(:link)- create_link(link, **attrs)- resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)- resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])[1]+ resolve("x #{create_embed(:link, **attrs)} y", maps:)end# Rendering a resolved target through the maps (the id is already known here;@@ -115,11 +120,11 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doendit "renders a tag target, honoring an import-time rename" do- maps = FakePlaceholderMaps.new(tag_name: { 3 => "shipped" })+ maps = FakePlaceholderMaps.new(tag_name: { 3 => "shipped" }, tag_id: { 3 => 30 })resolved = render({ url: "/tag/release", target_type: link_target::TAG, target_id: 3 }, maps:)- expect(resolved).to eq("x https://dest.example.com/tag/shipped y")+ expect(resolved).to eq("x https://dest.example.com/tag/shipped/30 y")endit "renders a category target as its slug path plus the destination id" do@@ -138,6 +143,260 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doexpect(resolved).to eq("x https://dest.example.com/c/support/billing/20 y")end+ it "remaps a canonical tag link by id even when its old slug names another tag" do+ create_tag(3, "release")+ create_tag(42, "support")+ maps =+ FakePlaceholderMaps.new(+ tag_name: {+ 3 => "release",+ 42 => "support-renamed",+ },+ tag_id: {+ 3 => 30,+ 42 => 420,+ },+ )+ token =+ create_embed(+ :link,+ url: "/tag/release/42/l/latest",+ target_type: link_target::TAG,+ target_id: 42,+ target_suffix: "/l/latest",+ original_markdown: '[Support](/tag/release/42/l/latest "title")',+ url_offset: 10,+ )++ expect(resolve(token, maps:)).to eq(+ '[Support](https://dest.example.com/tag/support-renamed/420/l/latest "title")',+ )+ end++ it "restores a canonical tag link when the destination id is unavailable" do+ maps = FakePlaceholderMaps.new(tag_name: { 42 => "support-renamed" })+ token =+ create_embed(+ :link,+ url: "/tag/support/42",+ target_type: link_target::TAG,+ target_id: 42,+ original_markdown: "[Support](/tag/support/42)",+ url_offset: 10,+ )+ resolver = described_class.new(intermediate_db, maps, owner_type:)++ expect(resolver.resolve_all([{ id: 1, raw: token }])[1]).to eq("[Support](/tag/support/42)")+ expect(resolver.unresolved_embeds.map(&:entity_id)).to eq([42])+ end++ it "reattaches a category's filter tail after the destination id" do+ maps =+ FakePlaceholderMaps.new(+ category_id: {+ 2 => 20,+ },+ category_slug_path: {+ 2 => "support:billing",+ },+ )++ resolved =+ render(+ {+ url: "/c/support/billing/2/l/latest",+ target_type: link_target::CATEGORY,+ target_id: 2,+ target_suffix: "/l/latest",+ },+ maps:,+ )++ expect(resolved).to eq("x https://dest.example.com/c/support/billing/20/l/latest y")+ end++ # A multi-tag route names several records; it is rebuilt only when the+ # category and every tag map, and any miss restores the source URL.+ describe "multi-tag routes" do+ let(:category_tag_maps) do+ FakePlaceholderMaps.new(+ category_id: {+ 2 => 20,+ },+ category_slug_path: {+ 2 => "addons:plugin",+ },+ tag_name: {+ 3 => "official-stuff",+ },+ )+ end++ before { create_tag(3, "official") }++ it "renders a category+tag target from the mapped coordinates" do+ resolved =+ render(+ {+ url: "/tags/c/plugin/2/official",+ target_type: link_target::CATEGORY_TAG,+ target_id: 2,+ target_tag_path: "official",+ },+ maps: category_tag_maps,+ )++ expect(resolved).to eq(+ "x https://dest.example.com/tags/c/addons/plugin/20/official-stuff y",+ )+ end++ it "keeps the subcategory filter and the suffix around the mapped parts" do+ resolved =+ render(+ {+ url: "/tags/c/plugin/2/none/official/l/top?period=yearly",+ target_type: link_target::CATEGORY_TAG,+ target_id: 2,+ target_tag_path: "none/official",+ target_suffix: "/l/top?period=yearly",+ },+ maps: category_tag_maps,+ )++ expect(resolved).to eq(+ "x https://dest.example.com/tags/c/addons/plugin/20/none/official-stuff/l/top?period=yearly y",+ )+ end++ it "resolves a legacy slug-path category the way category links do" do+ create_category(7, "howto")+ create_category(8, "devs", parent_category_id: 7)+ maps =+ FakePlaceholderMaps.new(+ category_id: {+ 8 => 80,+ },+ category_slug_path: {+ 8 => "guides:devs",+ },+ tag_name: {+ 3 => "official",+ },+ )++ resolved =+ render(+ {+ url: "/tags/c/howto/devs/official",+ target_type: link_target::CATEGORY_TAG,+ target_name: "howto:devs",+ target_tag_path: "official",+ },+ maps:,+ )++ expect(resolved).to eq("x https://dest.example.com/tags/c/guides/devs/80/official y")+ end++ it "renders an intersection target with every tag mapped" do+ create_tag(4, "wine")+ maps = FakePlaceholderMaps.new(tag_name: { 3 => "official", 4 => "vino" })++ resolved =+ render(+ {+ url: "/tags/intersection/official/wine",+ target_type: link_target::TAG_INTERSECTION,+ target_tag_path: "official/wine",+ },+ maps:,+ )++ expect(resolved).to eq("x https://dest.example.com/tags/intersection/official/vino y")+ end++ it "treats a lone none/all segment as the tag, not as a subcategory filter" do+ create_tag(5, "none")+ maps =+ FakePlaceholderMaps.new(+ category_id: {+ 2 => 20,+ },+ category_slug_path: {+ 2 => "addons:plugin",+ },+ tag_name: {+ 5 => "nothing",+ },+ )++ resolved =+ render(+ {+ url: "/tags/c/plugin/2/none",+ target_type: link_target::CATEGORY_TAG,+ target_id: 2,+ target_tag_path: "none",+ },+ maps:,+ )++ expect(resolved).to eq("x https://dest.example.com/tags/c/addons/plugin/20/nothing y")+ end++ it "falls back to the source URL when the category does not map" do+ maps = FakePlaceholderMaps.new(tag_name: { 3 => "official" })++ resolved =+ render(+ {+ url: "/tags/c/plugin/2/official",+ target_type: link_target::CATEGORY_TAG,+ target_id: 2,+ target_tag_path: "official",+ },+ maps:,+ )++ # The report itself flows through the shared unresolved-link branch,+ # covered below with the other miss cases.+ expect(resolved).to eq("x /tags/c/plugin/2/official y")+ end++ it "falls back whole when one of the intersection tags does not map" do+ create_tag(4, "wine")+ maps = FakePlaceholderMaps.new(tag_name: { 3 => "official" })++ resolved =+ render(+ {+ url: "/tags/intersection/official/wine",+ target_type: link_target::TAG_INTERSECTION,+ target_tag_path: "official/wine",+ },+ maps:,+ )++ expect(resolved).to eq("x /tags/intersection/official/wine y")+ end++ it "falls back when a tag name is not a source tag at all" do+ resolved =+ render(+ {+ url: "/tags/c/plugin/2/mystery",+ target_type: link_target::CATEGORY_TAG,+ target_id: 2,+ target_tag_path: "mystery",+ },+ maps: category_tag_maps,+ )++ expect(resolved).to eq("x /tags/c/plugin/2/mystery y")+ end+ end+it "renders a badge target with the destination id and slug" domaps = FakePlaceholderMaps.new(badge: { 9 => { id: 90, slug: "great-work" } })@@ -194,6 +453,55 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doexpect(resolved).to eq("x https://dest.example.com/u/new_bob y")end+ # A slug-only `/t/<slug>` link carries no id; the slug resolves against the+ # source topics, and only an unambiguous one rewrites.++ it "resolves a slug-only topic link when exactly one topic carries the slug" do+ create_topic(300, "how-to-fix-it")+ maps = FakePlaceholderMaps.new(topic_id: { 300 => 99 })++ resolved =+ render(+ {+ url: "/t/how-to-fix-it?page=2",+ target_type: link_target::TOPIC,+ target_name: "how-to-fix-it",+ target_suffix: "?page=2",+ },+ maps:,+ )++ expect(resolved).to eq("x https://dest.example.com/t/99?page=2 y")+ end++ it "restores the source URL for an unknown slug" do+ url = "https://old.example.com/t/ghost-topic"++ resolved =+ render({ url:, target_type: link_target::TOPIC, target_name: "ghost-topic" }, maps:)++ expect(resolved).to eq("x #{url} y")+ end++ it "restores the source URL when several topics share the slug" do+ create_topic(300, "dup-slug")+ create_topic(301, "dup-slug")+ link =+ create_embed(+ :link,+ url: "https://old.example.com/t/dup-slug",+ target_type: link_target::TOPIC,+ target_name: "dup-slug",+ )+ maps = FakePlaceholderMaps.new(topic_id: { 300 => 99, 301 => 100 })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])++ expect(resolved[1]).to eq("x https://old.example.com/t/dup-slug y")+ expect(resolver.unresolved_embeds.map(&:kind)).to eq([:link])+ end+it "resolves a category target by its parent:child slug path" docreate_category(1, "support")create_category(2, "billing", parent_category_id: 1)@@ -251,7 +559,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver docreate_tag(3, "release")create_tag(4, "releases")Migrations::Database::IntermediateDB::TagSynonym.create(synonym_tag_id: 4, target_tag_id: 3)- maps = FakePlaceholderMaps.new(tag_name: { 3 => "shipped" })+ maps = FakePlaceholderMaps.new(tag_name: { 3 => "shipped" }, tag_id: { 3 => 30 })resolved =render(@@ -259,20 +567,20 @@ RSpec.describe Migrations::Importer::PlaceholderResolver domaps:,)- expect(resolved).to eq("x https://dest.example.com/tag/shipped y")+ expect(resolved).to eq("x https://dest.example.com/tag/shipped/30 y")end# Reporting: an internal link that can't be resolved falls back to the source URL# but is still recorded, since a stale internal link points at the wrong record.it "falls back to the source URL and reports an unresolved internal link" do- link = placeholder.mint(:link)- create_link(- link,- url: "https://old.example.com/t/slug/300",- target_type: link_target::TOPIC,- target_id: 300,- )+ link =+ create_embed(+ :link,+ url: "https://old.example.com/t/slug/300",+ target_type: link_target::TOPIC,+ target_id: 300,+ )maps = FakePlaceholderMaps.new(post: { 1 => { topic_id: 42, post_number: 3 } })resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)@@ -290,8 +598,8 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doendit "reports the failing name when a named target can't be resolved" do- link = placeholder.mint(:link)- create_link(link, url: "/u/ghost", target_type: link_target::USER, target_name: "ghost")+ link =+ create_embed(:link, url: "/u/ghost", target_type: link_target::USER, target_name: "ghost")resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])@@ -299,8 +607,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doendit "does not report an external link that falls back" do- link = placeholder.mint(:link)- create_link(link, url: "https://elsewhere.example.com/page")+ link = create_embed(:link, url: "https://elsewhere.example.com/page")resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/rendering_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/rendering_spec.rbindex 6edbbe52a1a..938f16e4435 100644--- a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/rendering_spec.rb+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/rendering_spec.rb@@ -5,142 +5,161 @@ RSpec.describe Migrations::Importer::PlaceholderResolver dodescribe "rendering fallbacks" doit "keeps the source URL when the link target is unmapped" do- link = placeholder.mint(:link)- Migrations::Database::IntermediateDB::EmbedLink.create(- owner_type: embed_owner::POST,- owner_id: 1,- placeholder: link,- url: "https://old.example.com/x",- text: "See",- target_type: link_target::TOPIC,- target_id: 300,- )-- resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])+ link =+ create_embed(+ :link,+ url: "https://old.example.com/x",+ text: "See",+ target_type: link_target::TOPIC,+ target_id: 300,+ )- expect(resolved[1]).to eq("x [See](https://old.example.com/x) y")+ expect(resolve("x #{link} y")).to eq("x [See](https://old.example.com/x) y")endit "falls back to the recorded username when the user is unmapped" do- quote = placeholder.mint(:quote)- Migrations::Database::IntermediateDB::EmbedQuote.create(- owner_type: embed_owner::POST,- owner_id: 1,- placeholder: quote,- quoted_user_id: 5,- quoted_username: "ghost",- )-- resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])+ quote = create_embed(:quote, quoted_user_id: 5, quoted_username: "ghost")- expect(resolved[1]).to eq('x [quote="ghost"] y')+ expect(resolve("x #{quote} y")).to eq('x [quote="ghost"] y')endit "falls back to the recorded name when the user is unmapped" do- quote = placeholder.mint(:quote)- Migrations::Database::IntermediateDB::EmbedQuote.create(- owner_type: embed_owner::POST,- owner_id: 1,- placeholder: quote,- quoted_user_id: 5,- quoted_username: "ghost",- quoted_name: "Ghost User",- )+ quote =+ create_embed(:quote, quoted_user_id: 5, quoted_username: "ghost", quoted_name: "Ghost User")- resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])-- expect(resolved[1]).to eq('x [quote="Ghost User, username:ghost"] y')+ expect(resolve("x #{quote} y")).to eq('x [quote="Ghost User, username:ghost"] y')endit "omits the username: part when the name equals the username" do- quote = placeholder.mint(:quote)- Migrations::Database::IntermediateDB::EmbedQuote.create(- owner_type: embed_owner::POST,- owner_id: 1,- placeholder: quote,- quoted_user_id: 5,- quoted_username: "ghost",- quoted_name: "ghost",- )+ quote =+ create_embed(:quote, quoted_user_id: 5, quoted_username: "ghost", quoted_name: "ghost")++ expect(resolve("x #{quote} y")).to eq('x [quote="ghost"] y')+ end++ it "renders a bare [quote] when nothing identifies the quoted author" do+ quote = create_embed(:quote)++ expect(resolve("x #{quote} y")).to eq("x [quote] y")+ end++ it "keeps the resolved coordinates when nothing identifies the quoted author" do+ quote = create_embed(:quote, quoted_post_id: 200)+ maps = FakePlaceholderMaps.new(post: { 200 => { topic_id: 9, post_number: 4 } })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])- expect(resolved[1]).to eq('x [quote="ghost"] y')+ # The empty leading segment is what core's header parser needs to read+ # `post:`/`topic:` as coordinates instead of as the username.+ expect(resolved[1]).to eq('x [quote=", post:4, topic:9"] y')+ expect(resolver.unresolved_embeds).to be_emptyend- it "renders a bare [quote] when nothing identifies the quoted author" do- quote = placeholder.mint(:quote)- Migrations::Database::IntermediateDB::EmbedQuote.create(- owner_type: embed_owner::POST,- owner_id: 1,- placeholder: quote,- )+ it "rebuilds from what resolved and reports when the quoted post is unmapped" do+ quote = create_embed(:quote, quoted_post_id: 200, quoted_user_id: 5, quoted_username: "sam")+ maps = FakePlaceholderMaps.new(user: { 5 => { username: "sammy" } })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])- expect(resolved[1]).to eq("x [quote] y")+ expect(resolved[1]).to eq('x [quote="sammy"] y')+ expect(resolver.unresolved_embeds).to contain_exactly(+ described_class::UnresolvedEmbed.new(+ kind: :quote,+ entity_id: 200,+ owner_id: 1,+ owner_url: nil,+ ),+ )+ end++ it "reports the source coordinates of a quote that has no resolvable post" do+ # A `post:` with no `topic:` can never resolve to a source post, and the+ # source's own post number is meaningless on the destination.+ quote = create_embed(:quote, quoted_post_number: 2, quoted_username: "sam")++ expect(resolve("x #{quote} y")).to eq('x [quote="sam"] y')+ expect(resolver.unresolved_embeds.map(&:entity_id)).to eq(["post:2"])endit "drops an entity-backed embed whose markdown is unavailable" do- poll = placeholder.mint(:poll)- Migrations::Database::IntermediateDB::EmbedPoll.create(- owner_type: embed_owner::POST,- owner_id: 1,- placeholder: poll,- poll_id: 3,- )+ poll = create_embed(:poll, poll_id: 3)- resolved = resolver.resolve_all([{ id: 1, raw: "before #{poll} after" }])+ resolved = resolve("before #{poll} after")- expect(resolved[1]).to eq("before after")- expect(Migrations::Placeholder).not_to be_include(resolved[1])+ expect(resolved).to eq("before after")+ expect(Migrations::Placeholder).not_to be_include(resolved)endit "keeps backslashes and digits in replacement content verbatim" do- link = placeholder.mint(:link)- Migrations::Database::IntermediateDB::EmbedLink.create(- owner_type: embed_owner::POST,- owner_id: 1,- placeholder: link,- url: 'https://old.example.com/a\1b',- text: 'C:\temp\readme',- )-- resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])+ link = create_embed(:link, url: 'https://old.example.com/a\1b', text: 'C:\temp\readme')# A string-argument gsub would eat the backslashes and treat `\1` as a# backreference; the block form leaves the text byte-for-byte.- expect(resolved[1]).to eq('x [C:\temp\readme](https://old.example.com/a\1b) y')+ expect(resolve("x #{link} y")).to eq('x [C:\temp\readme](https://old.example.com/a\1b) y')endenddescribe "full-URL upload fallback" doit "puts the verbatim markdown back and still reports when the sha1 is unmapped" do- upload = placeholder.mint(:upload)snippet = ""- Migrations::Database::IntermediateDB::EmbedUpload.create(- owner_type: embed_owner::POST,- owner_id: 1,- placeholder: upload,- upload_id: "sha1",- original_markdown: snippet,- )+ upload = create_embed(:upload, upload_id: "sha1", original_markdown: snippet)++ expect(resolve("see #{upload} here")).to eq("see #{snippet} here")+ expect(resolver.unresolved_embeds.map(&:entity_id)).to eq(["sha1"])+ end- resolved = resolver.resolve_all([{ id: 1, raw: "see #{upload} here" }])+ it "declines a foreign-host row even when its sha1 maps, and restores the snippet" do+ snippet = ""+ upload =+ create_embed(+ :upload,+ upload_id: "sha1",+ original_markdown: snippet,+ external_host: "other-forum.example",+ )+ # The mapped markdown existing is exactly the dangerous case: a foreign+ # basename colliding with a source upload sha1.+ maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "" })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{upload} y" }])- expect(resolved[1]).to eq("see #{snippet} here")+ expect(resolved[1]).to eq("x #{snippet} y")expect(resolver.unresolved_embeds.map(&:entity_id)).to eq(["sha1"])end+ it "maps an external-host row when the host is explicitly trusted" do+ upload =+ create_embed(+ :upload,+ upload_id: "sha1",+ original_markdown: "",+ external_host: "cdn.example.com",+ )+ maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "" })+ resolver =+ described_class.new(+ intermediate_db,+ maps,+ owner_type: embed_owner::POST,+ trusted_upload_hosts: ["CDN.EXAMPLE.COM"],+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{upload} y" }])++ expect(resolved[1]).to eq("x  y")+ expect(resolver.unresolved_embeds).to be_empty+ end+it "prefers the mapped upload markdown over the verbatim snippet" do- upload = placeholder.mint(:upload)- Migrations::Database::IntermediateDB::EmbedUpload.create(- owner_type: embed_owner::POST,- owner_id: 1,- placeholder: upload,- upload_id: "sha1",- original_markdown: "",- )+ upload =+ create_embed(+ :upload,+ upload_id: "sha1",+ original_markdown: "",+ )maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "" })resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)@@ -151,6 +170,53 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doendend+ describe "bare upload destinations" do+ let(:maps) do+ FakePlaceholderMaps.new(+ upload: {+ "sha1" => {+ short_url: "upload://new.jpg",+ url: "https://dest.example.com/uploads/default/original/1X/new.jpg",+ },+ },+ upload_markdown: {+ "sha1" => "",+ },+ )+ end++ it "renders a reference definition's destination as the new short URL" do+ upload = create_embed(:upload, upload_id: "sha1", original_markdown: "upload://old.jpg")++ expect(resolve("[1]: #{upload}")).to eq("[1]: upload://new.jpg")+ expect(resolver.unresolved_embeds).to be_empty+ end++ it "renders a linkified full URL as the destination's full URL" do+ upload =+ create_embed(+ :upload,+ upload_id: "sha1",+ original_markdown: "https://old.example.com/uploads/default/original/1X/old.jpg",+ )++ expect(resolve("see #{upload} here")).to eq(+ "see https://dest.example.com/uploads/default/original/1X/new.jpg here",+ )+ end++ it "still renders a whole image construct as the destination's markdown" do+ upload =+ create_embed(+ :upload,+ upload_id: "sha1",+ original_markdown: "",+ )++ expect(resolve("x #{upload} y")).to eq("x  y")+ end+ end+describe "#unresolved_embeds" dolet(:maps) { FakePlaceholderMaps.new(post: { 100 => { topic_id: 42, post_number: 3 } }) }@@ -201,25 +267,18 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do)end- it "does not record entity-backed embeds that resolve" do- upload = placeholder.mint(:upload)- Migrations::Database::IntermediateDB::EmbedUpload.create(- owner_type: embed_owner::POST,- owner_id: 100,- placeholder: upload,- upload_id: "sha1",- )- maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "" })- resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)-- resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])-- expect(resolver.unresolved_embeds).to be_empty- end-it "does not record quotes, external links or mentions (they fall back to source values)" dolink = placeholder.mint(:link)mention = placeholder.mint(:mention)+ quote = placeholder.mint(:quote)+ # A quote that names no post loses nothing when its user is unmapped, so+ # it stays out of the report; only lost coordinates are reported.+ Migrations::Database::IntermediateDB::EmbedQuote.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: quote,+ quoted_username: "ghost",+ )Migrations::Database::IntermediateDB::EmbedLink.create(owner_type: embed_owner::POST,owner_id: 100,@@ -234,7 +293,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doname: "ghost",)- resolver.resolve_all([{ id: 100, raw: "#{link} #{mention}" }])+ resolver.resolve_all([{ id: 100, raw: "#{quote} #{link} #{mention}" }])expect(resolver.unresolved_embeds).to be_emptyend@@ -301,18 +360,9 @@ RSpec.describe Migrations::Importer::PlaceholderResolver dodescribe "a mention that renders to nothing" doit "drops the mention and records it" do- mention = placeholder.mint(:mention)- Migrations::Database::IntermediateDB::EmbedMention.create(- owner_type: embed_owner::POST,- owner_id: 1,- placeholder: mention,- mention_type: mention_type::USER,- target_id: 7,- )-- resolved = resolver.resolve_all([{ id: 1, raw: "hey #{mention} there" }])+ mention = create_embed(:mention, mention_type: mention_type::USER, target_id: 7)- expect(resolved[1]).to eq("hey there")+ expect(resolve("hey #{mention} there")).to eq("hey there")expect(resolver.unresolved_embeds).to contain_exactly(described_class::UnresolvedEmbed.new(kind: :mention,@@ -325,34 +375,32 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doenddescribe "custom emoji" do- def create_emoji(placeholder_token, name)- Migrations::Database::IntermediateDB::EmbedEmoji.create(- owner_type: embed_owner::POST,- owner_id: 1,- placeholder: placeholder_token,- name:,- )- end-it "renders a custom emoji, honoring an import-time rename" do- emoji = placeholder.mint(:emoji)- create_emoji(emoji, "parrot")+ emoji = create_embed(:emoji, name: "parrot")maps = FakePlaceholderMaps.new(emoji_name: { "parrot" => "party_parrot" })- resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)- resolved = resolver.resolve_all([{ id: 1, raw: "nice #{emoji} work" }])+ expect(resolve("nice #{emoji} work", maps:)).to eq("nice :party_parrot: work")+ end++ it "matches the map regardless of the case the author typed" do+ # The converter records the author's spelling; core lowercases a shortcode+ # before looking it up, so the map is keyed by the folded name.+ emoji = create_embed(:emoji, name: "MYEMOJI")+ maps = FakePlaceholderMaps.new(emoji_name: { "myemoji" => "my_emoji" })- expect(resolved[1]).to eq("nice :party_parrot: work")+ expect(resolve("hi #{emoji}", maps:)).to eq("hi :my_emoji:")end- it "falls back to the source name when the emoji is unmapped, without a report" do- emoji = placeholder.mint(:emoji)- create_emoji(emoji, "parrot")- resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)+ it "keeps the author's case on a miss" do+ emoji = create_embed(:emoji, name: "MyEmoji")- resolved = resolver.resolve_all([{ id: 1, raw: "hi #{emoji}" }])+ expect(resolve("hi #{emoji}")).to eq("hi :MyEmoji:")+ end++ it "falls back to the source name when the emoji is unmapped, without a report" do+ emoji = create_embed(:emoji, name: "parrot")- expect(resolved[1]).to eq("hi :parrot:")+ expect(resolve("hi #{emoji}")).to eq("hi :parrot:")expect(resolver.unresolved_embeds).to be_emptyendenddiff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/substitution_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/substitution_spec.rbindex 735afee5411..a251176cb7d 100644--- a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/substitution_spec.rb+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/substitution_spec.rb@@ -134,10 +134,6 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doexpect(resolver.orphan_placeholders.map(&:placeholder)).to eq([token])end- it "leaves a body untouched when it has no linkage rows" do- expect(resolver.resolve_all([{ id: 9, raw: "plain body" }])).to eq({ 9 => "plain body" })- end-it "issues no linkage queries when no owner in the batch has a token" doallow(intermediate_db).to receive(:query).and_call_originaldiff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/verbatim_fallback_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/verbatim_fallback_spec.rbnew file mode 100644index 00000000000..f137450af94--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/verbatim_fallback_spec.rb@@ -0,0 +1,388 @@+# frozen_string_literal: true++# The round-trip contract for embeds that carry their verbatim source: a+# resolution hit rewrites only what resolution changed (a link's destination),+# and any miss restores the exact source substring — never a canonical+# approximation that drops titles, spacing, or a suffix the author didn't+# write.+RSpec.describe Migrations::Importer::PlaceholderResolver do+ include_context "with placeholder resolver"++ describe "links with a verbatim source" do+ # A slug-only topic link resolves through a topic slug lookup; with no+ # topics to look up against, the titled construct restores byte-exact.+ it "restores a slug-only topic link byte-exact when the slug is unknown" do+ original = %{[read this](/t/how-to-fix-it "worth it")}+ token =+ create_embed(+ :link,+ original_markdown: original,+ url: "/t/how-to-fix-it",+ text: "read this",+ target_type: link_target::TOPIC,+ target_name: "how-to-fix-it",+ url_offset: 12,+ )++ expect(resolve("x #{token} y")).to eq("x #{original} y")+ end++ # A multi-tag route names several records; with nothing mapped the whole+ # construct restores byte-exact, filter and suffix included.+ it "restores a category+tag link byte-exact when nothing is mapped" do+ original =+ %{[official plugins](https://old.example.com/tags/c/plugin/22/none/official/l/top "tabs")}+ token =+ create_embed(+ :link,+ original_markdown: original,+ url: "https://old.example.com/tags/c/plugin/22/none/official/l/top",+ text: "official plugins",+ target_type: link_target::CATEGORY_TAG,+ target_id: 22,+ target_tag_path: "none/official",+ target_suffix: "/l/top",+ )++ expect(resolve("x #{token} y")).to eq("x #{original} y")+ end++ it "restores an intersection link byte-exact when a tag is unmapped" do+ url = "https://old.example.com/tags/intersection/food/wine"+ token =+ create_embed(+ :link,+ original_markdown: url,+ url:,+ target_type: link_target::TAG_INTERSECTION,+ target_tag_path: "food/wine",+ )++ expect(resolve("see #{token} there")).to eq("see #{url} there")+ end++ it "restores a titled link byte-exact when the target is unmapped" do+ original = %{[See](https://old.example.com/t/x/300 "why this matters")}+ token =+ create_embed(+ :link,+ original_markdown: original,+ url: "https://old.example.com/t/x/300",+ text: "See",+ target_type: link_target::TOPIC,+ target_id: 300,+ )++ expect(resolve("x #{token} y")).to eq("x #{original} y")+ end++ it "rewrites only the destination inside a titled link on a hit" do+ token =+ create_embed(+ :link,+ original_markdown: %{[See](https://old.example.com/t/x/300 "why this matters")},+ url: "https://old.example.com/t/x/300",+ url_offset: "[See](".bytesize,+ text: "See",+ target_type: link_target::TOPIC,+ target_id: 300,+ )++ resolved = resolve("x #{token} y", maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))++ expect(resolved).to eq(%{x [See](https://dest.example.com/t/12 "why this matters") y})+ end++ # `[](url)` — an empty label is valid CommonMark and stays empty: only+ # the destination span is rewritten, and there is no label span to touch.+ it "rewrites the destination of an empty-label link and keeps the label empty" do+ token =+ create_embed(+ :link,+ original_markdown: "[](https://old.example.com/t/x/300)",+ url: "https://old.example.com/t/x/300",+ url_offset: "[](".bytesize,+ text: "",+ target_type: link_target::TOPIC,+ target_id: 300,+ )++ resolved = resolve(token.to_s, maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))++ expect(resolved).to eq("[](https://dest.example.com/t/12)")+ end++ it "keeps angle brackets and padding around a rewritten destination" do+ token =+ create_embed(+ :link,+ original_markdown: "[x]( <https://old.example.com/t/x/300> )",+ url: "https://old.example.com/t/x/300",+ url_offset: "[x]( <".bytesize,+ text: "x",+ target_type: link_target::TOPIC,+ target_id: 300,+ )++ resolved = resolve(token.to_s, maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))++ expect(resolved).to eq("[x]( <https://dest.example.com/t/12> )")+ end++ it "rewrites both spellings of a self-link" do+ url = "https://old.example.com/t/x/300"+ token =+ create_embed(+ :link,+ original_markdown: "[#{url}](#{url})",+ url:,+ url_offset: url.bytesize + 3,+ label_url_offset: 1,+ text: url,+ target_type: link_target::TOPIC,+ target_id: 300,+ )++ resolved = resolve(token.to_s, maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))++ expect(resolved).to eq("[https://dest.example.com/t/12](https://dest.example.com/t/12)")+ end++ it "leaves a URL the author repeated in the title untouched on a hit" do+ url = "https://old.example.com/t/x/300"+ token =+ create_embed(+ :link,+ original_markdown: %{[docs](#{url} "mirror of #{url}")},+ url:,+ url_offset: "[docs](".bytesize,+ text: "docs",+ target_type: link_target::TOPIC,+ target_id: 300,+ )++ resolved = resolve(token.to_s, maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))++ # Only the recorded destination span changes; the title is the author's+ # text even when it happens to spell the same URL.+ expect(resolved).to eq(+ %{[docs](https://dest.example.com/t/12 "mirror of https://old.example.com/t/x/300")},+ )+ end++ it "rebuilds canonically on a hit when no destination span was recorded" do+ token =+ create_embed(+ :link,+ original_markdown: %{[See](https://old.example.com/t/x/300 "why this matters")},+ url: "https://old.example.com/t/x/300",+ text: "See",+ target_type: link_target::TOPIC,+ target_id: 300,+ )++ resolved = resolve(token.to_s, maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))++ # Without a span, a value search over the snippet could hit a title, so+ # the construct is rebuilt from its parts instead (losing the title, but+ # never rewriting the author's prose).+ expect(resolved).to eq("[See](https://dest.example.com/t/12)")+ end++ it "rebuilds canonically on a hit when the recorded span holds other bytes" do+ token =+ create_embed(+ :link,+ original_markdown: %{[See](https://old.example.com/t/x/300 "why this matters")},+ url: "https://old.example.com/t/x/300",+ # In bounds, but pointing at the label instead of the destination. A+ # splice there would write the new URL into the author's prose.+ url_offset: 1,+ text: "See",+ target_type: link_target::TOPIC,+ target_id: 300,+ )++ resolved = resolve(token.to_s, maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))++ expect(resolved).to eq("[See](https://dest.example.com/t/12)")+ end++ it "splices only the destination when the label span holds other bytes" do+ url = "https://old.example.com/t/x/300"+ token =+ create_embed(+ :link,+ original_markdown: "[docs](#{url})",+ url:,+ url_offset: "[docs](".bytesize,+ # In bounds, wrong position: the label span is dropped, the+ # destination span still applies.+ label_url_offset: 1,+ text: "docs",+ target_type: link_target::TOPIC,+ target_id: 300,+ )++ resolved = resolve(token.to_s, maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))++ expect(resolved).to eq("[docs](https://dest.example.com/t/12)")+ end++ it "restores an external link byte-exact" do+ original = %{[docs](https://elsewhere.example.org/page "docs")}+ token =+ create_embed(+ :link,+ original_markdown: original,+ url: "https://elsewhere.example.org/page",+ text: "docs",+ )++ expect(resolve(token.to_s)).to eq(original)+ end+ end++ describe "broadcast mentions" do+ it "keeps the author's spelling when the destination names nothing" do+ here = create_embed(:mention, mention_type: mention_type::HERE, name: "Here")+ all = create_embed(:mention, mention_type: mention_type::ALL, name: "ALL")++ resolved = resolve("#{here} and #{all}", maps: FakePlaceholderMaps.new(here_mention: nil))++ expect(resolved).to eq("@Here and @ALL")+ end++ it "keeps the author's spelling when the destination's here-mention is the same name" do+ # Here-matching is case-insensitive at cook time, so `@Here` works on a+ # destination configured with "here"; canonicalizing it would change+ # bytes for nothing.+ token = create_embed(:mention, mention_type: mention_type::HERE, name: "Here")++ expect(resolve("cc #{token}")).to eq("cc @Here")+ end++ it "remaps a here-mention that the destination names differently" do+ token = create_embed(:mention, mention_type: mention_type::HERE, name: "Here")++ resolved = resolve("cc #{token}", maps: FakePlaceholderMaps.new(here_mention: "hier"))++ expect(resolved).to eq("cc @hier")+ end+ end++ describe "hashtags with a verbatim source" do+ it "keeps a bare hashtag bare when the resolved category cannot be rendered" do+ # The import resolved a type (mutating hashtag_type) but the destination+ # map misses: the source never wrote `::category`, so none may appear.+ token =+ create_embed(+ :hashtag,+ original_markdown: "#support",+ name: "support",+ hashtag_type: hashtag_type::CATEGORY,+ target_id: 7,+ )++ expect(resolve("see #{token}")).to eq("see #support")+ end++ it "restores a source-written suffix exactly on a miss" do+ token =+ create_embed(+ :hashtag,+ original_markdown: "#support::tag",+ name: "support",+ hashtag_type: hashtag_type::TAG,+ )++ expect(resolve(token.to_s)).to eq("#support::tag")+ end+ end++ describe "quotes with a verbatim source" do+ it "restores the exact opening tag when nothing resolved" do+ # Header parameters the reference columns don't model survive a miss.+ original = "[quote=bob, custom:keep-me]"+ token = create_embed(:quote, original_markdown: original, quoted_username: "bob")++ expect(resolve("#{token}\ntext\n[/quote]")).to eq("#{original}\ntext\n[/quote]")+ end++ it "restores the source tag and reports when the quoted post did not resolve" do+ # The user resolving is not enough: dropping `post:`/`topic:` would leave+ # an attribution pointing at no post at all.+ original = %{[quote="bob, post:2, topic:3"]}+ token =+ create_embed(+ :quote,+ original_markdown: original,+ quoted_user_id: 5,+ quoted_username: "bob",+ quoted_post_id: 200,+ )++ maps = FakePlaceholderMaps.new(user: { 5 => { username: "robert" } })+ resolver = described_class.new(intermediate_db, maps, owner_type:)++ resolved = resolver.resolve_all([{ id: 1, raw: token.to_s }])[1]++ expect(resolved).to eq(original)+ expect(resolver.unresolved_embeds.map(&:kind)).to eq(%i[quote])+ end++ it "rebuilds canonically once the quoted user resolved" do+ token =+ create_embed(+ :quote,+ original_markdown: %{[QUOTE="bob"]},+ quoted_user_id: 5,+ quoted_username: "bob",+ )++ resolved =+ resolve(token.to_s, maps: FakePlaceholderMaps.new(user: { 5 => { username: "robert" } }))++ expect(resolved).to eq(%{[quote="robert"]})+ end+ end++ describe "uploads with a verbatim source" do+ it "restores the exact source markup when the upload is unmapped" do+ original = ""+ token = create_embed(:upload, upload_id: "abcdef", original_markdown: original)++ expect(resolve(token.to_s)).to eq(original)+ expect(resolver.unresolved_embeds.map(&:kind)).to eq(%i[upload])+ end++ # A failed image paste (`[label|meta](upload://…)`, link position, no `!`)+ # rides the standard upload row: verbatim back on a miss, the+ # destination's own markdown on a hit — same as every other upload form.+ it "round-trips a link-position pipe label through miss and hit" do+ original = "[image|281x500](upload://abcdef.jpeg)"+ token = create_embed(:upload, upload_id: "abcdef", original_markdown: original)++ expect(resolve("x #{token} y")).to eq("x #{original} y")++ hit = FakePlaceholderMaps.new(upload_markdown: { "abcdef" => "" })+ expect(resolve("x #{token} y", maps: hit)).to eq("x  y")+ end++ it "restores a reference definition's destination when the upload is unmapped" do+ original = "upload://abcdef.jpg"+ token = create_embed(:upload, upload_id: "abcdef", original_markdown: original)++ expect(resolve("[1]: #{token}")).to eq("[1]: #{original}")+ expect(resolver.unresolved_embeds.map(&:kind)).to eq(%i[upload])+ end++ it "restores a prose full URL when the upload is unmapped" do+ original = "https://old.example.com/uploads/default/original/1X/abcdef.jpg"+ token = create_embed(:upload, upload_id: "abcdef", original_markdown: original)++ expect(resolve("see #{token} here")).to eq("see #{original} here")+ expect(resolver.unresolved_embeds.map(&:entity_id)).to eq(["abcdef"])+ end+ end+enddiff --git a/migrations/importer/spec/support/placeholder_resolver_context.rb b/migrations/importer/spec/support/placeholder_resolver_context.rbindex dd9cc4d7f1b..c9a4b24b64a 100644--- a/migrations/importer/spec/support/placeholder_resolver_context.rb+++ b/migrations/importer/spec/support/placeholder_resolver_context.rb@@ -15,12 +15,14 @@ class FakePlaceholderMapsgroup_nameposttopic_id+ uploadupload_markdownpoll_markdownevent_markdowncategory_slug_pathcategory_idtag_name+ tag_idbadgeemoji_name].each { |name| define_method(name) { |key| (@lookups[name] || {})[key] } }@@ -50,6 +52,56 @@ RSpec.shared_context "with placeholder resolver" dolet(:maps) { FakePlaceholderMaps.new }let(:owner_type) { embed_owner::POST }+ # Creates an embed row of `kind` for owner 1 and returns its placeholder+ # token, so an example is only its row attributes and expectation.+ def create_embed(kind, **attributes)+ token = placeholder.mint(kind)+ Migrations::Database::IntermediateDB.const_get("Embed#{kind.capitalize}").create(+ owner_type:,+ owner_id: 1,+ placeholder: token,+ **attributes,+ )+ token+ end++ # The source records a resolver example looks a recorded name up against.++ def create_user(original_id, username)+ Migrations::Database::IntermediateDB::User.create(+ original_id:,+ username:,+ created_at: Time.now,+ trust_level: 0,+ )+ end++ def create_category(original_id, slug, parent_category_id: nil)+ Migrations::Database::IntermediateDB::Category.create(+ original_id:,+ name: slug,+ slug:,+ parent_category_id:,+ user_id: 1,+ )+ end++ def create_tag(original_id, name)+ Migrations::Database::IntermediateDB::Tag.create(original_id:, name:, slug: name)+ end++ def create_topic(original_id, slug)+ Migrations::Database::IntermediateDB::Topic.create(original_id:, title: slug, slug:)+ end++ # Resolves one owner-1 body; `maps:` builds a fresh resolver over those+ # lookups, without it the shared subject resolves (so an example can inspect+ # `resolver.unresolved_embeds` afterwards).+ def resolve(raw, maps: nil)+ instance = maps ? described_class.new(intermediate_db, maps, owner_type:) : resolver+ instance.resolve_all([{ id: 1, raw: }])[1]+ end+around do |example|Dir.mktmpdir do |dir|db_path = File.join(dir, "intermediate.db")diff --git a/migrations/tooling/config/schema/intermediate_db/enums/link_target.rb b/migrations/tooling/config/schema/intermediate_db/enums/link_target.rbindex 76676256db0..0e72b133986 100644--- a/migrations/tooling/config/schema/intermediate_db/enums/link_target.rb+++ b/migrations/tooling/config/schema/intermediate_db/enums/link_target.rb@@ -8,4 +8,7 @@ Migrations::Tooling::Schema.enum :link_target dovalue :tag, 5value :group, 6value :badge, 7+ value :site, 8+ value :category_tag, 9+ value :tag_intersection, 10enddiff --git a/migrations/tooling/config/schema/intermediate_db/ignored.rb b/migrations/tooling/config/schema/intermediate_db/ignored.rbindex c40eac91eac..b2b843c815b 100644--- a/migrations/tooling/config/schema/intermediate_db/ignored.rb+++ b/migrations/tooling/config/schema/intermediate_db/ignored.rb@@ -172,7 +172,9 @@ Migrations::Tooling::Schema.ignored do:bookmarks,:browser_pageview_crawler_daily_rollups,:browser_pageview_event_scores,+ :browser_pageview_event_scores_backup,:browser_pageview_events,+ :browser_pageview_events_backup,:browser_pageview_session_engagement_daily_rollups,:browser_pageview_session_engagements,:custom_emojis,@@ -193,6 +195,15 @@ Migrations::Tooling::Schema.ignored do:invited_users,:invites,:javascript_caches,+ :mcp_audit_logs,+ :mcp_group_scopes,+ :mcp_oauth_access_tokens,+ :mcp_oauth_authorization_codes,+ :mcp_oauth_authorization_scopes,+ :mcp_oauth_authorizations,+ :mcp_oauth_clients,+ :mcp_oauth_refresh_tokens,+ :mcp_primitives,:message_bus,:nested_hot_post_scores,:nested_hot_score_snapshots,diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_hashtags.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_hashtags.rbindex ac761b2a8b9..c94407a1af6 100644--- a/migrations/tooling/config/schema/intermediate_db/tables/embed_hashtags.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_hashtags.rb@@ -28,6 +28,9 @@ Migrations::Tooling::Schema.table :embed_hashtags doadd_column :hashtag_type, :integer, enum: :hashtag_typeadd_column :target_id, :numericadd_column :name, :text, required: true+ # The verbatim source snippet, restored unchanged when the importer cannot+ # map the embed to a destination record (the round-trip fallback).+ add_column :original_markdown, :textindex :owner_type, :owner_idenddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_links.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_links.rbindex 7f2c7ff5702..90dd7b54a3b 100644--- a/migrations/tooling/config/schema/intermediate_db/tables/embed_links.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_links.rb@@ -9,13 +9,14 @@# `Migrations::Placeholder`. `owner_type`/`owner_id` name that owning record.## `target_type` is the kind of Discourse entity the link points at, a `link_target`-# value (`topic`, `post`, `user`, `category`, `tag`, `group` or `badge`); nil for an-# external link. The entity is identified in one of three forms, and only one is set+# value (`topic`, `post`, `user`, `category`, `tag`, `group`, `badge`, or the+# multi-coordinate `category_tag` / `tag_intersection`); nil for an external link. The entity is identified in one of three forms, and only one is set# per row: by id (`target_id`, the source `original_id`), by name (`target_name`),# or by coordinates (`target_topic_id` + `target_post_number`). Which form a row# uses follows from what the URL carries — with a Discourse source:## * topic `/t/a-topic/123`, `/t/123` -> target_id: 123+# * topic `/t/a-topic` (slug only) -> target_name: "a-topic"# * post `/p/456` -> target_id: 456# * post `/t/a-topic/123/7` -> target_topic_id: 123, target_post_number: 7# * user `/u/sam`, `/users/sam` -> target_name: "sam"@@ -27,14 +28,31 @@## The table shows what a Discourse source's URLs carry; the storage itself is less# strict. Any target type may be recorded by id — a converter that knows the-# target's `original_id` can skip the name. The name form resolves only for user,-# group, tag and category (a topic, post or badge has no name lookup), and-# coordinates only for a post.+# target's `original_id` can skip the name. The name form resolves for user,+# group, tag and category, and for a topic by its slug when exactly one source+# topic carries it (a post or badge has no name lookup); coordinates only for a+# post.## A category slug path is stored with a `:` separator, matching the hashtag# resolution maps so the importer reuses them verbatim. Post coordinates mirror# `embed_quotes`: post numbers are recomputed at import, so the importer resolves# them to a post rather than preserving them.+# The multi-tag routes name several records, so they carry a second coordinate in+# `target_tag_path` (tag names come from URL path segments, so `/` can't occur+# inside one):+#+# * category+tag `/tags/c/plugin/22/official` -> target_id: 22,+# target_tag_path: "official"+# * category+tag `/tags/c/howto/devs/import` -> target_name: "howto:devs",+# target_tag_path: "import"+# * category+tag `/tags/c/theme/61/none/extra` -> target_id: 61,+# target_tag_path: "none/extra"+# (`none`/`all` is core's+# subcategory filter, kept so+# the rebuilt route filters+# the same way)+# * intersection `/tags/intersection/wine/food` -> target_tag_path: "wine/food"+## `target_suffix` is everything after the matched route (further path, query string,# fragment), reattached verbatim when the URL is rebuilt: `/u/sam/summary` yields# target_name: "sam" plus target_suffix: "/summary", and `/t/a-topic/123?page=2`@@ -52,7 +70,18 @@ Migrations::Tooling::Schema.table :embed_links doadd_column :target_name, :textadd_column :target_topic_id, :numericadd_column :target_post_number, :integer+ add_column :target_tag_path, :textadd_column :target_suffix, :text+ # The verbatim source snippet, restored unchanged when the importer cannot+ # map the embed to a destination record (the round-trip fallback).+ add_column :original_markdown, :text+ # The destination's byte offset within `original_markdown` (the span is+ # `url`'s byte length): on a hit the importer rewrites exactly that span, so+ # the same URL repeated in a link title stays the author's text. A self-link+ # whose label spells the destination too records the label's spelling at+ # `label_url_offset`, rewritten alongside.+ add_column :url_offset, :integer+ add_column :label_url_offset, :integerindex :owner_type, :owner_idenddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_mentions.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_mentions.rbindex 4ac0eaa29aa..435482bcb2d 100644--- a/migrations/tooling/config/schema/intermediate_db/tables/embed_mentions.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_mentions.rb@@ -20,6 +20,9 @@ Migrations::Tooling::Schema.table :embed_mentions doadd_column :mention_type, :integer, enum: :mention_typeadd_column :target_id, :numericadd_column :name, :text+ # The verbatim source snippet, restored unchanged when the importer cannot+ # map the embed to a destination record (the round-trip fallback).+ add_column :original_markdown, :textindex :owner_type, :owner_idenddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_quotes.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_quotes.rbindex 5b808fc712c..146f5a7f130 100644--- a/migrations/tooling/config/schema/intermediate_db/tables/embed_quotes.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_quotes.rb@@ -25,6 +25,9 @@ Migrations::Tooling::Schema.table :embed_quotes doadd_column :quoted_user_id, :numericadd_column :quoted_username, :textadd_column :quoted_name, :text+ # The verbatim source snippet, restored unchanged when the importer cannot+ # map the embed to a destination record (the round-trip fallback).+ add_column :original_markdown, :textindex :owner_type, :owner_idenddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rbindex 610238d18cf..e6767704588 100644--- a/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb@@ -6,11 +6,17 @@# markdown (a post body today, a user bio etc. later); see `Migrations::Placeholder`.# `owner_type`/`owner_id` name that owning record.#-# `original_markdown` is the verbatim source snippet for an upload referenced by a-# full URL (e.g. `/uploads/.../<sha1>.png`), nil for a short `upload://` reference.-# When the sha1 maps to no Discourse upload, the importer puts this snippet back-# unchanged instead of dropping the embed — a hotlink to another forum's upload then-# survives as-is.+# `original_markdown` is the verbatim source snippet for both a full URL+# (`/uploads/.../<sha1>.png`) and a short `upload://` reference. When the id maps+# to no Discourse upload, the importer puts this snippet back unchanged instead+# of dropping the embed — a hotlink to another forum's upload then survives as-is.+#+# `external_host` is set when a full-URL upload points at a host the conversion does+# not recognize as the source's own. A foreign URL whose 40-hex basename happens to+# collide with a source upload sha1 must not be rewritten to the imported file, so+# `PlaceholderResolver` maps such a row only when `trusted_upload_hosts` explicitly+# includes the host (a CDN or old object-storage domain) — otherwise the verbatim+# snippet is restored.## NOTE: `upload_id` is `:text`, not `:numeric`. Upload `original_id`s in the# IntermediateDB are content hashes (see `uploads.id`, also `:text`). This matches@@ -21,6 +27,7 @@ Migrations::Tooling::Schema.table :embed_uploads doadd_column :owner_type, :integer, required: true, enum: :embed_owneradd_column :owner_id, :numeric, required: true+ add_column :external_host, :textadd_column :placeholder, :text, required: trueadd_column :upload_id, :textadd_column :original_markdown, :textdiff --git a/migrations/tooling/config/schema/intermediate_db/tables/topics.rb b/migrations/tooling/config/schema/intermediate_db/tables/topics.rbindex d4365216697..fccfde036cd 100644--- a/migrations/tooling/config/schema/intermediate_db/tables/topics.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/topics.rb@@ -2,6 +2,14 @@Migrations::Tooling::Schema.table :topics doindex :archetype+ index :slug++ # The destination regenerates its own slug from the title, but the SOURCE+ # slug is what a slug-only `/t/<slug>` link in a post body carries — the+ # resolver looks it up here to remap such links. Nullable because not every+ # source has slugs; resolution then finds nothing and the link stays as+ # written.+ column :slug, required: falseignore :bumped_at,:excerpt,@@ -26,7 +34,6 @@ Migrations::Tooling::Schema.table :topics do:reply_count,:reviewable_score,:score,- :slug,:spam_count,:word_count,reason: "Calculated columns"diff --git a/migrations/tooling/config/schema/plugin_manifest.yml b/migrations/tooling/config/schema/plugin_manifest.ymlindex be110c77ac7..702fc7a71fa 100644--- a/migrations/tooling/config/schema/plugin_manifest.yml+++ b/migrations/tooling/config/schema/plugin_manifest.yml@@ -45,7 +45,11 @@ plugins:user_options:- chat_announce_new_messages- chat_channel_list_filter+ - chat_channel_list_filter_dms+ - chat_channel_list_filter_starred- chat_channel_list_sort+ - chat_channel_list_sort_dms+ - chat_channel_list_sort_starred- chat_email_frequency- chat_enabled- chat_header_indicator_preference@@ -90,6 +94,7 @@ plugins:- ai_tool_secret_bindings- ai_tools- ai_topics_embeddings+ - ask_ai_logs- chat_message_custom_prompts- classification_results- completion_prompts@@ -135,6 +140,7 @@ plugins:- discourse_calendar_disabled_holidays- discourse_calendar_post_event_dates- discourse_post_event_events+ - discourse_post_event_hosts- discourse_post_event_invitees- livestream_topic_chat_channelscolumns: {}@@ -245,18 +251,18 @@ plugins:- voice_sessionscolumns: {}plugin_checksums:- automation: e69972690991bc04d92b2d161aac8921- boards: 3f7be99d5aaac4cf66318ef036e231a2- chat: ab866e1148b3ab93d658a44dc4417951+ automation: d168107063ed75d7e568f1c1250866f8+ boards: 7196f10413674b3c08480ff2aa42b2e8+ chat: 9c46fca8e1650c913c2b249268020d77discourse-adplugin: 9e52419decf1099c5bc301a90e6cc91adiscourse-affiliate: '08c6e1a851441a082e9be9ddddc63910'- discourse-ai: 1dbe6a6155758187d0a34e7fb0b24181+ discourse-ai: 8153214bb8d90e69b19638d7c908b5bcdiscourse-assign: 7a7a36e8ac0214e8a6aca1a1ed9899aadiscourse-cakeday: a5bb91366c8dc8d267088eb0c2679187discourse-captcha: fd538c1338062354276ccfda4cbafd55discourse-chat-integration: ffa3bf5dba9089491d2fc02971eb96f8discourse-data-explorer: fb71ab79dbe07af481ba147ea526406f- discourse-events: 6a305b06fa9121f17245cefbc726ce20+ discourse-events: 6837af88d62287f3a503c1267e091425discourse-gamification: 633256e241b030fddcc059076f7474f8discourse-github: 3bd17b2c09828e232e7565edbf2c746fdiscourse-graphviz: 8a1835520fbf8f7f5d1ef8a25d737e20@@ -271,15 +277,15 @@ plugin_checksums:discourse-reactions: e9de82d2ebb48eb434be2b646c7fec26discourse-rewind: 1e0f2acf6e8409f2d2f011e14405fa06discourse-rss-polling: 526db84c3d2ade0fa03497031fea2333- discourse-solved: 67743e14f4c5c2fae4487193c374e429+ discourse-solved: 3ebebf25c6eebc06b980f3bb02b4fa89discourse-subscriptions: 05eb12877a6a842a290a98fa0e70d462discourse-templates: a131c89a3e2b765f1eb6ec811c0fecea- discourse-topic-voting: 5464e186402b9589d71f338c704a7d21+ discourse-topic-voting: 9d0535677407702fa312965e20ac8fcbdiscourse-user-notes: 4c1e04fac5708b51c1b09bd8ed0d786cdiscourse-workflows: 61c0d4b94a17ad401490893981b23dd4discourse-zendesk-plugin: b78b305590ffedda38455574a19fdd38poll: e481f2cfa0e7c22778dee12a1e9c33c6styleguide: 0fd92e5ef0c99ee047f206961a587544- voice: 23e33508822e87035b4bee0925d26eff+ voice: cf9c46b63d1ee4c208e822201838cd52failed_plugins: []incomplete: falsediff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rbindex 42d5d1a120d..4d93d2481f6 100644--- a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb@@ -194,7 +194,7 @@ RSpec.describe Migrations::Tooling::Schema::DSL::Differ, :rails doexpect(names).to contain_exactly("created_at", "email")end- it "uses source table name when attributing unconfigured columns for copied tables" do+ it "uses source table name for unconfigured columns in copied tables" domanifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)allow(manifest).to receive(:available?).and_return(true)allow(manifest).to receive(:plugin_for_table).and_return(nil)