A platform for community discussion. Free, open, simple.
DEV: Add HashtagRewriter
Régis Hanol09340f6947be
diff --git a/frontend/discourse-markdown-it/src/features/hashtag-autocomplete.js b/frontend/discourse-markdown-it/src/features/hashtag-autocomplete.jsindex 174318ab9bb..bbba646e5fb 100644--- a/frontend/discourse-markdown-it/src/features/hashtag-autocomplete.js+++ b/frontend/discourse-markdown-it/src/features/hashtag-autocomplete.js@@ -4,6 +4,45 @@// not provide an accurate lookup for hashtags without a ::type suffix in those// cases if there are conflcting types of resources with the same slug.+import { hasAllowedBoundaries } from "./text-post-process";++const MATCHER =+ /(?<!\/)#([\u00C0-\u1FFF\u2C00-\uD7FF\w:-](?:[\u00C0-\u1FFF\u2C00-\uD7FF\w:.-]{0,99}[\u00C0-\u1FFF\u2C00-\uD7FF\w:-])?)/;++export function scanHashtags(raw) {+ const found = [];+ const matcher = new RegExp(MATCHER.source, "g");+ let match;++ while ((match = matcher.exec(raw))) {+ const end = match.index + match[0].length;+ if (!hasAllowedBoundaries(raw, match.index, end)) {+ continue;+ }++ found.push({ index: match.index, length: match[0].length, ref: match[1] });+ }++ return found;+}++export function spliceHashtags(raw, replacements) {+ let spliced = "";+ let last = 0;++ scanHashtags(raw).forEach((candidate, i) => {+ const replacement = replacements[i];+ if (replacement === undefined || replacement === null) {+ return;+ }++ spliced += raw.slice(last, candidate.index) + "#" + replacement;+ last = candidate.index + candidate.length;+ });++ return spliced + raw.slice(last);+}+function addHashtag(buffer, matches, state) {const options = state.md.options.discourse;const slug = matches[1];@@ -112,8 +151,7 @@ function addIconPlaceholder(buffer, state) {export function setup(helper) {helper.registerPlugin((md) => {const rule = {- matcher:- /(?<!\/)#([\u00C0-\u1FFF\u2C00-\uD7FF\w:-](?:[\u00C0-\u1FFF\u2C00-\uD7FF\w:.-]{0,99}[\u00C0-\u1FFF\u2C00-\uD7FF\w:-])?)/,+ matcher: MATCHER,onMatch: addHashtag,};diff --git a/frontend/discourse-markdown-it/src/features/text-post-process.js b/frontend/discourse-markdown-it/src/features/text-post-process.jsindex fe7559b3374..5fc3693f55a 100644--- a/frontend/discourse-markdown-it/src/features/text-post-process.js+++ b/frontend/discourse-markdown-it/src/features/text-post-process.js@@ -1,5 +1,8 @@+import MarkdownIt from "markdown-it";import { applyDataAttributes, parseBBCodeTag } from "./bbcode-block";+const { isPunctChar, isWhiteSpace } = new MarkdownIt().utils;+export class TextPostProcessRuler {constructor() {this.rules = [];@@ -76,11 +79,21 @@ export class TextPostProcessRuler {}}-function allowedBoundary(content, index, utils) {+function allowedBoundary(content, index) {let code = content.charCodeAt(index);- return (- utils.isWhiteSpace(code) || utils.isPunctChar(String.fromCharCode(code))- );+ return isWhiteSpace(code) || isPunctChar(String.fromCharCode(code));+}++export function hasAllowedBoundaries(content, start, end) {+ if (start > 0 && !allowedBoundary(content, start - 1)) {+ return false;+ }++ if (end < content.length && !allowedBoundary(content, end)) {+ return false;+ }++ return true;}function textPostProcess(content, state, ruler) {@@ -96,20 +109,10 @@ function textPostProcess(content, state, ruler) {break;}- // check boundary- if (match.index > 0) {- if (!allowedBoundary(content, match.index - 1, state.md.utils)) {- continue;- }- }-- // check forward boundary as well- if (match.index + match[0].length < content.length) {- if (- !allowedBoundary(content, match.index + match[0].length, state.md.utils)- ) {- continue;- }+ if (+ !hasAllowedBoundaries(content, match.index, match.index + match[0].length)+ ) {+ continue;}result = result || [];diff --git a/frontend/pretty-text-processor/pretty-text-ruby-interface.js b/frontend/pretty-text-processor/pretty-text-ruby-interface.jsindex ef53b4cea28..dff0fbd9ea4 100644--- a/frontend/pretty-text-processor/pretty-text-ruby-interface.js+++ b/frontend/pretty-text-processor/pretty-text-ruby-interface.js@@ -9,6 +9,10 @@ import * as avatarUtils from "discourse/lib/avatar-utils";import loadPluginFeatures from "discourse/static/markdown-it/features";import DiscourseMarkdownIt from "discourse-markdown-it";import { resetTranslationTree } from "discourse-markdown-it/features/emoji";+import {+ scanHashtags,+ spliceHashtags,+} from "discourse-markdown-it/features/hashtag-autocomplete";import { runtime } from "./runtime-state.js";// Passed to the markdown pipeline as plain callbacks, so functions not methods.@@ -139,4 +143,12 @@ export class PrettyTextRubyInterface {static performEmojiEscape(text, options) {return performEmojiEscape(text, options);}++ static scanHashtags(raw) {+ return scanHashtags(raw).map((candidate) => candidate.ref);+ }++ static spliceHashtags(raw, replacements) {+ return spliceHashtags(raw, replacements);+ }}diff --git a/lib/hashtag_rewriter.rb b/lib/hashtag_rewriter.rbnew file mode 100644index 00000000000..8396737ea35--- /dev/null+++ b/lib/hashtag_rewriter.rb@@ -0,0 +1,103 @@+# frozen_string_literal: true++class HashtagRewriter+ def self.sql_like_pattern(ref)+ "%##{ActiveRecord::Base.sanitize_sql_like(ref)}%"+ end++ def self.usable_ref?(ref)+ return false if ref.blank?++ PrettyText.scan_hashtags(" ##{ref} ") == [ref]+ end++ def initialize(raw, cook_options = {})+ @raw = raw.to_s+ @cook_options = cook_options+ end++ def rewrite+ wanted = {}++ refs.each_with_index do |ref, index|+ replacement = yield(ref)+ wanted[index] = replacement if replacement.present?+ end+ return @raw if wanted.empty?++ live = probed_as_hashtag(wanted)+ return @raw if live.empty?++ faithful = faithful_subset(live)+ return @raw if faithful.empty?++ splice(faithful)+ end++ private++ def refs+ @refs ||= PrettyText.scan_hashtags(@raw)+ end++ def splice(replacements)+ PrettyText.splice_hashtags(@raw, replacements)+ end++ def cooked(raw)+ PrettyText.markdown(raw, @cook_options.dup)+ end++ def probed_as_hashtag(wanted)+ markers = wanted.keys.index_with { |index| marker(index) }+ probed = Nokogiri::HTML5.fragment(cooked(splice(markers)))+ spans = probed.css("span.hashtag-raw").map(&:text).to_set++ wanted.select { |index, _| spans.include?("##{markers[index]}") }+ end++ def faithful_subset(live)+ return live if faithful?(live)++ kept = {}++ live.each do |index, replacement|+ kept[index] = replacement if faithful?(kept.merge(index => replacement))+ end++ kept+ end++ def faithful?(replacements)+ @baseline ||= hashtag_blind(cooked(@raw))++ hashtag_blind(cooked(splice(replacements))) == @baseline+ end++ def hashtag_blind(html)+ doc = Nokogiri::HTML5.fragment(html)++ doc.css("span.hashtag-raw, a.hashtag-cooked").each { |node| node.replace(nonce) }++ doc+ .css("a.anchor")+ .each do |node|+ node["name"] = ""+ node["href"] = ""+ end++ doc.to_html+ end++ def marker(index)+ "#{nonce}#{index}z"+ end++ def nonce+ @nonce ||=+ loop do+ candidate = "z#{SecureRandom.hex(6)}z"+ break candidate if @raw.exclude?(candidate)+ end+ end+enddiff --git a/lib/pretty_text.rb b/lib/pretty_text.rbindex 6f176324799..0c25f80cff1 100644--- a/lib/pretty_text.rb+++ b/lib/pretty_text.rb@@ -275,6 +275,14 @@ module PrettyTextprotect { v8.call("__PrettyText.sanitize", html.to_s, opts) }end+ def self.scan_hashtags(raw)+ protect { v8.call("__PrettyText.scanHashtags", raw.to_s) }+ end++ def self.splice_hashtags(raw, replacements)+ protect { v8.call("__PrettyText.spliceHashtags", raw.to_s, replacements) }+ end+def self.unescape_emoji(title)return title unless SiteSetting.enable_emoji? && titlediff --git a/spec/lib/hashtag_rewriter_spec.rb b/spec/lib/hashtag_rewriter_spec.rbnew file mode 100644index 00000000000..41635d14b89--- /dev/null+++ b/spec/lib/hashtag_rewriter_spec.rb@@ -0,0 +1,223 @@+# frozen_string_literal: true++RSpec.describe HashtagRewriter do+ def rewrite(raw, from: "alpha", to: "beta", cook_options: {})+ described_class.new(raw, cook_options).rewrite { |ref| to if ref.casecmp?(from) }+ end++ it "rewrites a plain reference" do+ expect(rewrite("See #alpha here.")).to eq("See #beta here.")+ end++ it "leaves references the markdown pipeline would not cook" do+ protected_raws = {+ "fenced code" => "Live #alpha\n\n```\nfenced #alpha\n```",+ "fenced code with a language" => "Live #alpha\n\n```ruby\nfenced #alpha\n```",+ "tilde fence" => "Live #alpha\n\n~~~\nfenced #alpha\n~~~",+ "unclosed fence" => "Live #alpha\n\n```\nfenced #alpha",+ "inline code" => "Live #alpha and `inline #alpha`",+ "double backtick span" => "Live #alpha and ``inline #alpha``",+ "indented code" => "Live #alpha\n\n indented #alpha",+ "code bbcode" => "Live #alpha and [code]bbcode #alpha[/code]",+ "link text" => "Live #alpha and [#alpha](https://example.com/)",+ "link destination" => "Live #alpha and [x](https://example.com/#alpha)",+ "autolink" => "Live #alpha and <https://example.com/#alpha>",+ "linkified url fragment" => "Live #alpha and https://example.com/?tab=#alpha",+ "image alt" => "Live #alpha and ",+ "html attribute" => %{Live #alpha and <div data-x="#alpha">x</div>},+ }++ protected_raws.each do |name, raw|+ expect(rewrite(raw)).to eq(raw.sub("Live #alpha", "Live #beta")),+ "expected #{name} to be left alone"+ end+ end++ it "rewrites references the markdown pipeline does cook" do+ live_raws = {+ "blockquote" => "> quoted #alpha",+ "quote bbcode" => "[quote]quoted #alpha[/quote]",+ "heading" => "## heading #alpha",+ "list item" => "- item #alpha",+ "table cell" => "| a |\n| --- |\n| #alpha |",+ "bold" => "**bold #alpha**",+ "details" => "[details=x]inner #alpha[/details]",+ "unclosed backtick" => "unclosed `#alpha",+ }++ live_raws.each do |name, raw|+ expect(rewrite(raw)).to eq(raw.sub("#alpha", "#beta")), "expected #{name} to be rewritten"+ end+ end++ it "matches whole references, so a longer reference is untouched" do+ expect(rewrite("I use #alpha.js daily and #alpha rarely.")).to eq(+ "I use #alpha.js daily and #beta rarely.",+ )+ end++ it "rewrites references bounded by unicode symbols, like the cooker does" do+ expect(rewrite("€#alpha and #alpha")).to eq("€#beta and #beta")+ expect(rewrite("#alpha™ and #alpha")).to eq("#beta™ and #beta")+ end++ it "leaves character references alone" do+ expect(rewrite("hi #123 and {", from: "123", to: "456")).to eq("hi #456 and {")+ expect(rewrite("hi #x1f600 and 😀", from: "x1f600", to: "grin")).to eq(+ "hi #grin and 😀",+ )+ end++ it "rewrites lookalikes the cooker does not read as character references" do+ expect(rewrite("{", from: "123", to: "456")).to eq("Lj")+ expect(rewrite("&#123;", from: "123", to: "456")).to eq("&#456;")+ end++ it "leaves reference link labels alone" do+ labelled_raws = {+ "full reference" => "Here is #alpha\n\n[link][#alpha]\n\n[#alpha]: https://example.com/",+ "collapsed reference" => "Here is #alpha\n\n[#alpha][]\n\n[#alpha]: https://example.com/",+ "shortcut reference" => "Here is #alpha\n\n[#alpha]\n\n[#alpha]: https://example.com/",+ "image reference" => "Here is #alpha\n\n![x][#alpha]\n\n[#alpha]: https://example.com/a.png",+ }++ labelled_raws.each do |name, raw|+ expect(rewrite(raw)).to eq(raw.sub("Here is #alpha", "Here is #beta")),+ "expected #{name} labels to be left alone"+ end+ end++ it "rewrites a bracketed reference that resolves to no link" do+ expect(rewrite("[#alpha] fix crash")).to eq("[#beta] fix crash")+ end++ it "rewrites inside a dead reference link, whose text stays visible" do+ raw = "[see #alpha here][#u1]\n\n[#u2]: https://example.com/"++ expect(rewrite(raw)).to eq("[see #beta here][#u1]\n\n[#u2]: https://example.com/")+ end++ it "leaves footnote labels alone" do+ raw = "a [^#alpha] b\n\n[^#alpha]: note"++ expect(rewrite(raw)).to eq(raw)+ end++ it "leaves a reference the linkifier consumes alone" do+ expect(rewrite("see #docs.example.com now", from: "docs.example.com", to: "guides")).to eq(+ "see #docs.example.com now",+ )+ end++ it "leaves a reference the emphasis parser consumes alone" do+ expect(rewrite("x #_alpha_ y", from: "_alpha_", to: "under")).to eq("x #_alpha_ y")+ end++ it "leaves a reference the typographer splits alone" do+ expect(rewrite("x #alpha--z y", from: "alpha--z", to: "dashed")).to eq("x #alpha--z y")+ end++ it "leaves occurrences that render nothing alone" do+ fence = "```#alpha\ncode\n```"+ orphan_definition = "[#alpha]: https://example.com/"++ expect(rewrite(fence)).to eq(fence)+ expect(rewrite(orphan_definition)).to eq(orphan_definition)+ end++ it "does not read a non-ascii neighbour as part of an ascii reference" do+ expect(rewrite("One #alpha and two #alphaé here.")).to eq("One #beta and two #alphaé here.")+ end++ it "matches case-insensitively when the caller asks it to" do+ expect(rewrite("See #ALPHA here.")).to eq("See #beta here.")+ end++ it "handles a raw with every context at once" do+ raw = <<~MD+ Live #alpha here.++ ```+ fence #alpha+ ```++ Inline `#alpha` span.++ [#alpha](https://example.com/) and https://example.com/?tab=#alpha++ indented #alpha++ <div data-x="#alpha">html</div>++ > quoted #alpha is live+ MD++ result = rewrite(raw)++ expect(result.scan("#beta").size).to eq(2)+ expect(result).to include("Live #beta here.", "> quoted #beta is live")+ expect(result).to include("fence #alpha", "`#alpha`", "?tab=#alpha", " indented #alpha")+ end++ it "respects the cook options it is given" do+ raw = "and `inline #alpha` here"++ expect(rewrite(raw)).to eq(raw)+ expect(rewrite(raw, cook_options: { markdown_it_rules: %w[table] })).to eq(+ "and `inline #beta` here",+ )+ end++ it "leaves everything alone when the block never asks for a replacement" do+ raw = "See #alpha and #gamma here."++ expect(described_class.new(raw).rewrite { nil }).to eq(raw)+ end++ describe "parity with the markdown pipeline" do+ it "agrees with the cooker about which references become hashtags" do+ Fabricate(:tag, name: "alpha")++ corpus = [+ "plain #alpha",+ "#alpha.js and #alpha",+ "a#alpha and #alpha",+ "/#alpha and #alpha",+ "x:#alpha",+ "##alpha",+ "(#alpha)",+ "€#alpha and #alpha™",+ "#alpha, #alpha and #alpha",+ ]++ corpus.each do |raw|+ cooked = PrettyText.cook(raw)+ anchors = cooked.scan(/class="hashtag-cooked"/).size+ matched = PrettyText.scan_hashtags(raw).count { |ref| ref == "alpha" }++ expect(matched).to eq(anchors), "mismatch for #{raw.inspect}"+ end+ end+ end++ describe ".sql_like_pattern" do+ it "escapes LIKE wildcards" do+ expect(described_class.sql_like_pattern("a_b%c\\d")).to eq("%#a\\_b\\%c\\\\d%")+ end+ end++ describe ".usable_ref?" do+ it "accepts references the matcher can read back" do+ expect(described_class.usable_ref?("node.js")).to eq(true)+ expect(described_class.usable_ref?("café")).to eq(true)+ expect(described_class.usable_ref?("support:bucks")).to eq(true)+ end++ it "rejects references that would not survive a round trip" do+ expect(described_class.usable_ref?("")).to eq(false)+ expect(described_class.usable_ref?(nil)).to eq(false)+ expect(described_class.usable_ref?("has space")).to eq(false)+ expect(described_class.usable_ref?("trailing.")).to eq(false)+ end+ end+end