Skip to content

A platform for community discussion. Free, open, simple.

DEV: Add HashtagRewriter

Régis Hanol09340f6947be
diff --git a/frontend/discourse-markdown-it/src/features/hashtag-autocomplete.js b/frontend/discourse-markdown-it/src/features/hashtag-autocomplete.js
index 174318ab9bb..bbba646e5fb 100644
--- a/frontend/discourse-markdown-it/src/features/hashtag-autocomplete.js
+++ b/frontend/discourse-markdown-it/src/features/hashtag-autocomplete.js
@@ -4,6 +4,45 @@
// not provide an accurate lookup for hashtags without a ::type suffix in those
// cases if there are conflcting types of resources with the same slug.
+import { hasAllowedBoundaries } from "./text-post-process";
+
+const MATCHER =
+ /(?<!\/)#([\u00C0-\u1FFF\u2C00-\uD7FF\w:-](?:[\u00C0-\u1FFF\u2C00-\uD7FF\w:.-]{0,99}[\u00C0-\u1FFF\u2C00-\uD7FF\w:-])?)/;
+
+export function scanHashtags(raw) {
+ const found = [];
+ const matcher = new RegExp(MATCHER.source, "g");
+ let match;
+
+ while ((match = matcher.exec(raw))) {
+ const end = match.index + match[0].length;
+ if (!hasAllowedBoundaries(raw, match.index, end)) {
+ continue;
+ }
+
+ found.push({ index: match.index, length: match[0].length, ref: match[1] });
+ }
+
+ return found;
+}
+
+export function spliceHashtags(raw, replacements) {
+ let spliced = "";
+ let last = 0;
+
+ scanHashtags(raw).forEach((candidate, i) => {
+ const replacement = replacements[i];
+ if (replacement === undefined || replacement === null) {
+ return;
+ }
+
+ spliced += raw.slice(last, candidate.index) + "#" + replacement;
+ last = candidate.index + candidate.length;
+ });
+
+ return spliced + raw.slice(last);
+}
+
function addHashtag(buffer, matches, state) {
const options = state.md.options.discourse;
const slug = matches[1];
@@ -112,8 +151,7 @@ function addIconPlaceholder(buffer, state) {
export function setup(helper) {
helper.registerPlugin((md) => {
const rule = {
- matcher:
- /(?<!\/)#([\u00C0-\u1FFF\u2C00-\uD7FF\w:-](?:[\u00C0-\u1FFF\u2C00-\uD7FF\w:.-]{0,99}[\u00C0-\u1FFF\u2C00-\uD7FF\w:-])?)/,
+ matcher: MATCHER,
onMatch: addHashtag,
};
diff --git a/frontend/discourse-markdown-it/src/features/text-post-process.js b/frontend/discourse-markdown-it/src/features/text-post-process.js
index fe7559b3374..5fc3693f55a 100644
--- a/frontend/discourse-markdown-it/src/features/text-post-process.js
+++ b/frontend/discourse-markdown-it/src/features/text-post-process.js
@@ -1,5 +1,8 @@
+import MarkdownIt from "markdown-it";
import { applyDataAttributes, parseBBCodeTag } from "./bbcode-block";
+const { isPunctChar, isWhiteSpace } = new MarkdownIt().utils;
+
export class TextPostProcessRuler {
constructor() {
this.rules = [];
@@ -76,11 +79,21 @@ export class TextPostProcessRuler {
}
}
-function allowedBoundary(content, index, utils) {
+function allowedBoundary(content, index) {
let code = content.charCodeAt(index);
- return (
- utils.isWhiteSpace(code) || utils.isPunctChar(String.fromCharCode(code))
- );
+ return isWhiteSpace(code) || isPunctChar(String.fromCharCode(code));
+}
+
+export function hasAllowedBoundaries(content, start, end) {
+ if (start > 0 && !allowedBoundary(content, start - 1)) {
+ return false;
+ }
+
+ if (end < content.length && !allowedBoundary(content, end)) {
+ return false;
+ }
+
+ return true;
}
function textPostProcess(content, state, ruler) {
@@ -96,20 +109,10 @@ function textPostProcess(content, state, ruler) {
break;
}
- // check boundary
- if (match.index > 0) {
- if (!allowedBoundary(content, match.index - 1, state.md.utils)) {
- continue;
- }
- }
-
- // check forward boundary as well
- if (match.index + match[0].length < content.length) {
- if (
- !allowedBoundary(content, match.index + match[0].length, state.md.utils)
- ) {
- continue;
- }
+ if (
+ !hasAllowedBoundaries(content, match.index, match.index + match[0].length)
+ ) {
+ continue;
}
result = result || [];
diff --git a/frontend/pretty-text-processor/pretty-text-ruby-interface.js b/frontend/pretty-text-processor/pretty-text-ruby-interface.js
index ef53b4cea28..dff0fbd9ea4 100644
--- a/frontend/pretty-text-processor/pretty-text-ruby-interface.js
+++ b/frontend/pretty-text-processor/pretty-text-ruby-interface.js
@@ -9,6 +9,10 @@ import * as avatarUtils from "discourse/lib/avatar-utils";
import loadPluginFeatures from "discourse/static/markdown-it/features";
import DiscourseMarkdownIt from "discourse-markdown-it";
import { resetTranslationTree } from "discourse-markdown-it/features/emoji";
+import {
+ scanHashtags,
+ spliceHashtags,
+} from "discourse-markdown-it/features/hashtag-autocomplete";
import { runtime } from "./runtime-state.js";
// Passed to the markdown pipeline as plain callbacks, so functions not methods.
@@ -139,4 +143,12 @@ export class PrettyTextRubyInterface {
static performEmojiEscape(text, options) {
return performEmojiEscape(text, options);
}
+
+ static scanHashtags(raw) {
+ return scanHashtags(raw).map((candidate) => candidate.ref);
+ }
+
+ static spliceHashtags(raw, replacements) {
+ return spliceHashtags(raw, replacements);
+ }
}
diff --git a/lib/hashtag_rewriter.rb b/lib/hashtag_rewriter.rb
new file mode 100644
index 00000000000..8396737ea35
--- /dev/null
+++ b/lib/hashtag_rewriter.rb
@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+class HashtagRewriter
+ def self.sql_like_pattern(ref)
+ "%##{ActiveRecord::Base.sanitize_sql_like(ref)}%"
+ end
+
+ def self.usable_ref?(ref)
+ return false if ref.blank?
+
+ PrettyText.scan_hashtags(" ##{ref} ") == [ref]
+ end
+
+ def initialize(raw, cook_options = {})
+ @raw = raw.to_s
+ @cook_options = cook_options
+ end
+
+ def rewrite
+ wanted = {}
+
+ refs.each_with_index do |ref, index|
+ replacement = yield(ref)
+ wanted[index] = replacement if replacement.present?
+ end
+ return @raw if wanted.empty?
+
+ live = probed_as_hashtag(wanted)
+ return @raw if live.empty?
+
+ faithful = faithful_subset(live)
+ return @raw if faithful.empty?
+
+ splice(faithful)
+ end
+
+ private
+
+ def refs
+ @refs ||= PrettyText.scan_hashtags(@raw)
+ end
+
+ def splice(replacements)
+ PrettyText.splice_hashtags(@raw, replacements)
+ end
+
+ def cooked(raw)
+ PrettyText.markdown(raw, @cook_options.dup)
+ end
+
+ def probed_as_hashtag(wanted)
+ markers = wanted.keys.index_with { |index| marker(index) }
+ probed = Nokogiri::HTML5.fragment(cooked(splice(markers)))
+ spans = probed.css("span.hashtag-raw").map(&:text).to_set
+
+ wanted.select { |index, _| spans.include?("##{markers[index]}") }
+ end
+
+ def faithful_subset(live)
+ return live if faithful?(live)
+
+ kept = {}
+
+ live.each do |index, replacement|
+ kept[index] = replacement if faithful?(kept.merge(index => replacement))
+ end
+
+ kept
+ end
+
+ def faithful?(replacements)
+ @baseline ||= hashtag_blind(cooked(@raw))
+
+ hashtag_blind(cooked(splice(replacements))) == @baseline
+ end
+
+ def hashtag_blind(html)
+ doc = Nokogiri::HTML5.fragment(html)
+
+ doc.css("span.hashtag-raw, a.hashtag-cooked").each { |node| node.replace(nonce) }
+
+ doc
+ .css("a.anchor")
+ .each do |node|
+ node["name"] = ""
+ node["href"] = ""
+ end
+
+ doc.to_html
+ end
+
+ def marker(index)
+ "#{nonce}#{index}z"
+ end
+
+ def nonce
+ @nonce ||=
+ loop do
+ candidate = "z#{SecureRandom.hex(6)}z"
+ break candidate if @raw.exclude?(candidate)
+ end
+ end
+end
diff --git a/lib/pretty_text.rb b/lib/pretty_text.rb
index 6f176324799..0c25f80cff1 100644
--- a/lib/pretty_text.rb
+++ b/lib/pretty_text.rb
@@ -275,6 +275,14 @@ module PrettyText
protect { v8.call("__PrettyText.sanitize", html.to_s, opts) }
end
+ def self.scan_hashtags(raw)
+ protect { v8.call("__PrettyText.scanHashtags", raw.to_s) }
+ end
+
+ def self.splice_hashtags(raw, replacements)
+ protect { v8.call("__PrettyText.spliceHashtags", raw.to_s, replacements) }
+ end
+
def self.unescape_emoji(title)
return title unless SiteSetting.enable_emoji? && title
diff --git a/spec/lib/hashtag_rewriter_spec.rb b/spec/lib/hashtag_rewriter_spec.rb
new file mode 100644
index 00000000000..41635d14b89
--- /dev/null
+++ b/spec/lib/hashtag_rewriter_spec.rb
@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+RSpec.describe HashtagRewriter do
+ def rewrite(raw, from: "alpha", to: "beta", cook_options: {})
+ described_class.new(raw, cook_options).rewrite { |ref| to if ref.casecmp?(from) }
+ end
+
+ it "rewrites a plain reference" do
+ expect(rewrite("See #alpha here.")).to eq("See #beta here.")
+ end
+
+ it "leaves references the markdown pipeline would not cook" do
+ protected_raws = {
+ "fenced code" => "Live #alpha\n\n```\nfenced #alpha\n```",
+ "fenced code with a language" => "Live #alpha\n\n```ruby\nfenced #alpha\n```",
+ "tilde fence" => "Live #alpha\n\n~~~\nfenced #alpha\n~~~",
+ "unclosed fence" => "Live #alpha\n\n```\nfenced #alpha",
+ "inline code" => "Live #alpha and `inline #alpha`",
+ "double backtick span" => "Live #alpha and ``inline #alpha``",
+ "indented code" => "Live #alpha\n\n indented #alpha",
+ "code bbcode" => "Live #alpha and [code]bbcode #alpha[/code]",
+ "link text" => "Live #alpha and [#alpha](https://example.com/)",
+ "link destination" => "Live #alpha and [x](https://example.com/#alpha)",
+ "autolink" => "Live #alpha and <https://example.com/#alpha>",
+ "linkified url fragment" => "Live #alpha and https://example.com/?tab=#alpha",
+ "image alt" => "Live #alpha and ![#alpha](https://example.com/a.png)",
+ "html attribute" => %{Live #alpha and <div data-x="#alpha">x</div>},
+ }
+
+ protected_raws.each do |name, raw|
+ expect(rewrite(raw)).to eq(raw.sub("Live #alpha", "Live #beta")),
+ "expected #{name} to be left alone"
+ end
+ end
+
+ it "rewrites references the markdown pipeline does cook" do
+ live_raws = {
+ "blockquote" => "> quoted #alpha",
+ "quote bbcode" => "[quote]quoted #alpha[/quote]",
+ "heading" => "## heading #alpha",
+ "list item" => "- item #alpha",
+ "table cell" => "| a |\n| --- |\n| #alpha |",
+ "bold" => "**bold #alpha**",
+ "details" => "[details=x]inner #alpha[/details]",
+ "unclosed backtick" => "unclosed `#alpha",
+ }
+
+ live_raws.each do |name, raw|
+ expect(rewrite(raw)).to eq(raw.sub("#alpha", "#beta")), "expected #{name} to be rewritten"
+ end
+ end
+
+ it "matches whole references, so a longer reference is untouched" do
+ expect(rewrite("I use #alpha.js daily and #alpha rarely.")).to eq(
+ "I use #alpha.js daily and #beta rarely.",
+ )
+ end
+
+ it "rewrites references bounded by unicode symbols, like the cooker does" do
+ expect(rewrite("€#alpha and #alpha")).to eq("€#beta and #beta")
+ expect(rewrite("#alpha™ and #alpha")).to eq("#beta™ and #beta")
+ end
+
+ it "leaves character references alone" do
+ expect(rewrite("hi #123 and &#123;", from: "123", to: "456")).to eq("hi #456 and &#123;")
+ expect(rewrite("hi #x1f600 and &#x1f600;", from: "x1f600", to: "grin")).to eq(
+ "hi #grin and &#x1f600;",
+ )
+ end
+
+ it "rewrites lookalikes the cooker does not read as character references" do
+ expect(rewrite("&#123", from: "123", to: "456")).to eq("&#456")
+ expect(rewrite("&amp;#123;", from: "123", to: "456")).to eq("&amp;#456;")
+ end
+
+ it "leaves reference link labels alone" do
+ labelled_raws = {
+ "full reference" => "Here is #alpha\n\n[link][#alpha]\n\n[#alpha]: https://example.com/",
+ "collapsed reference" => "Here is #alpha\n\n[#alpha][]\n\n[#alpha]: https://example.com/",
+ "shortcut reference" => "Here is #alpha\n\n[#alpha]\n\n[#alpha]: https://example.com/",
+ "image reference" => "Here is #alpha\n\n![x][#alpha]\n\n[#alpha]: https://example.com/a.png",
+ }
+
+ labelled_raws.each do |name, raw|
+ expect(rewrite(raw)).to eq(raw.sub("Here is #alpha", "Here is #beta")),
+ "expected #{name} labels to be left alone"
+ end
+ end
+
+ it "rewrites a bracketed reference that resolves to no link" do
+ expect(rewrite("[#alpha] fix crash")).to eq("[#beta] fix crash")
+ end
+
+ it "rewrites inside a dead reference link, whose text stays visible" do
+ raw = "[see #alpha here][#u1]\n\n[#u2]: https://example.com/"
+
+ expect(rewrite(raw)).to eq("[see #beta here][#u1]\n\n[#u2]: https://example.com/")
+ end
+
+ it "leaves footnote labels alone" do
+ raw = "a [^#alpha] b\n\n[^#alpha]: note"
+
+ expect(rewrite(raw)).to eq(raw)
+ end
+
+ it "leaves a reference the linkifier consumes alone" do
+ expect(rewrite("see #docs.example.com now", from: "docs.example.com", to: "guides")).to eq(
+ "see #docs.example.com now",
+ )
+ end
+
+ it "leaves a reference the emphasis parser consumes alone" do
+ expect(rewrite("x #_alpha_ y", from: "_alpha_", to: "under")).to eq("x #_alpha_ y")
+ end
+
+ it "leaves a reference the typographer splits alone" do
+ expect(rewrite("x #alpha--z y", from: "alpha--z", to: "dashed")).to eq("x #alpha--z y")
+ end
+
+ it "leaves occurrences that render nothing alone" do
+ fence = "```#alpha\ncode\n```"
+ orphan_definition = "[#alpha]: https://example.com/"
+
+ expect(rewrite(fence)).to eq(fence)
+ expect(rewrite(orphan_definition)).to eq(orphan_definition)
+ end
+
+ it "does not read a non-ascii neighbour as part of an ascii reference" do
+ expect(rewrite("One #alpha and two #alphaé here.")).to eq("One #beta and two #alphaé here.")
+ end
+
+ it "matches case-insensitively when the caller asks it to" do
+ expect(rewrite("See #ALPHA here.")).to eq("See #beta here.")
+ end
+
+ it "handles a raw with every context at once" do
+ raw = <<~MD
+ Live #alpha here.
+
+ ```
+ fence #alpha
+ ```
+
+ Inline `#alpha` span.
+
+ [#alpha](https://example.com/) and https://example.com/?tab=#alpha
+
+ indented #alpha
+
+ <div data-x="#alpha">html</div>
+
+ > quoted #alpha is live
+ MD
+
+ result = rewrite(raw)
+
+ expect(result.scan("#beta").size).to eq(2)
+ expect(result).to include("Live #beta here.", "> quoted #beta is live")
+ expect(result).to include("fence #alpha", "`#alpha`", "?tab=#alpha", " indented #alpha")
+ end
+
+ it "respects the cook options it is given" do
+ raw = "and `inline #alpha` here"
+
+ expect(rewrite(raw)).to eq(raw)
+ expect(rewrite(raw, cook_options: { markdown_it_rules: %w[table] })).to eq(
+ "and `inline #beta` here",
+ )
+ end
+
+ it "leaves everything alone when the block never asks for a replacement" do
+ raw = "See #alpha and #gamma here."
+
+ expect(described_class.new(raw).rewrite { nil }).to eq(raw)
+ end
+
+ describe "parity with the markdown pipeline" do
+ it "agrees with the cooker about which references become hashtags" do
+ Fabricate(:tag, name: "alpha")
+
+ corpus = [
+ "plain #alpha",
+ "#alpha.js and #alpha",
+ "a#alpha and #alpha",
+ "/#alpha and #alpha",
+ "x:#alpha",
+ "##alpha",
+ "(#alpha)",
+ "€#alpha and #alpha™",
+ "#alpha, #alpha and #alpha",
+ ]
+
+ corpus.each do |raw|
+ cooked = PrettyText.cook(raw)
+ anchors = cooked.scan(/class="hashtag-cooked"/).size
+ matched = PrettyText.scan_hashtags(raw).count { |ref| ref == "alpha" }
+
+ expect(matched).to eq(anchors), "mismatch for #{raw.inspect}"
+ end
+ end
+ end
+
+ describe ".sql_like_pattern" do
+ it "escapes LIKE wildcards" do
+ expect(described_class.sql_like_pattern("a_b%c\\d")).to eq("%#a\\_b\\%c\\\\d%")
+ end
+ end
+
+ describe ".usable_ref?" do
+ it "accepts references the matcher can read back" do
+ expect(described_class.usable_ref?("node.js")).to eq(true)
+ expect(described_class.usable_ref?("café")).to eq(true)
+ expect(described_class.usable_ref?("support:bucks")).to eq(true)
+ end
+
+ it "rejects references that would not survive a round trip" do
+ expect(described_class.usable_ref?("")).to eq(false)
+ expect(described_class.usable_ref?(nil)).to eq(false)
+ expect(described_class.usable_ref?("has space")).to eq(false)
+ expect(described_class.usable_ref?("trailing.")).to eq(false)
+ end
+ end
+end