Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Improve emoji, footnotes, and polls in Markdown output (#43920)

Penar Musaraje6528f198926
diff --git a/config/initializers/markdown_endpoints.rb b/config/initializers/markdown_endpoints.rb
index e0428bc2633..e0e4a316c83 100644
--- a/config/initializers/markdown_endpoints.rb
+++ b/config/initializers/markdown_endpoints.rb
@@ -4,6 +4,7 @@ require "reverse_markdown"
require "markdown_endpoint/accept_header"
require "markdown_endpoint/request_constraint"
require "markdown_endpoint/vary_middleware"
+require "markdown_endpoint/emoji_converter"
require "markdown_endpoint/cooked_processor"
require "markdown_endpoint/topic_renderer"
require "markdown_endpoint/topic_list_renderer"
diff --git a/lib/markdown_endpoint/cooked_processor.rb b/lib/markdown_endpoint/cooked_processor.rb
index 82a8ec2ba01..383505eb36e 100644
--- a/lib/markdown_endpoint/cooked_processor.rb
+++ b/lib/markdown_endpoint/cooked_processor.rb
@@ -4,7 +4,7 @@ module MarkdownEndpoint
class CookedProcessor
BLOCK_TAG = "discourse-markdown-block"
INLINE_TAG = "discourse-markdown-inline"
- VERSION = 2
+ VERSION = 4
class PreservedBlockConverter < ReverseMarkdown::Converters::Base
def convert(node, _state = {})
@@ -21,12 +21,13 @@ module MarkdownEndpoint
ReverseMarkdown::Converters.register(BLOCK_TAG.to_sym, PreservedBlockConverter.new)
ReverseMarkdown::Converters.register(INLINE_TAG.to_sym, PreservedInlineConverter.new)
- def self.to_markdown(cooked_html)
- new(cooked_html).to_markdown
+ def self.to_markdown(cooked_html, post_url: nil)
+ new(cooked_html, post_url:).to_markdown
end
- def initialize(cooked_html)
+ def initialize(cooked_html, post_url: nil)
@fragment = Nokogiri::HTML5.fragment(cooked_html.to_s)
+ @post_url = post_url
end
def to_markdown
@@ -36,6 +37,7 @@ module MarkdownEndpoint
replace_hashtags
replace_lightboxes
replace_code_blocks
+ replace_footnotes
replace_quotes
replace_oneboxes
replace_details
@@ -54,7 +56,10 @@ module MarkdownEndpoint
def replace_emojis
@fragment
.css("img.emoji")
- .each { |image| image.replace(preserved_inline(image["title"] || image["alt"] || "")) }
+ .each do |image|
+ shortcode = image["title"] || image["alt"] || ""
+ image.replace(preserved_inline(EmojiConverter.convert(shortcode)))
+ end
end
def replace_mentions
@@ -108,6 +113,13 @@ module MarkdownEndpoint
end
end
+ def replace_footnotes
+ @fragment.css("a.footnote-backref").each(&:remove)
+ @fragment
+ .css("sup.footnote-ref a")
+ .each { |anchor| anchor.replace(preserved_inline(escape_text(anchor.text))) }
+ end
+
def replace_quotes
@fragment
.css("aside.quote")
@@ -172,8 +184,10 @@ module MarkdownEndpoint
.each do |poll|
title =
poll["data-poll-title"].presence ||
- poll.at_css("[data-poll-title]")&.[]("data-poll-title").presence || "Poll"
- poll.replace(preserved_block("_Poll: #{escape_text(title)} (view on site)_"))
+ poll.at_css("[data-poll-title]")&.[]("data-poll-title").presence
+ label = title.present? ? "Poll: #{escape_text(title)}" : "Poll"
+ label += " ([view on site](#{absolute_url(@post_url)}))" if @post_url.present?
+ poll.replace(preserved_block("_#{label}_"))
end
end
@@ -196,7 +210,7 @@ module MarkdownEndpoint
end
def convert_html(html)
- fragment = self.class.new(html)
+ fragment = self.class.new(html, post_url: @post_url)
fragment.to_markdown
end
diff --git a/lib/markdown_endpoint/emoji_converter.rb b/lib/markdown_endpoint/emoji_converter.rb
new file mode 100644
index 00000000000..de3b9f1f03c
--- /dev/null
+++ b/lib/markdown_endpoint/emoji_converter.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module MarkdownEndpoint
+ class EmojiConverter
+ def self.convert(text)
+ text
+ .to_s
+ .gsub(Emoji::EMOJI_CODE_REGEXP) do |shortcode|
+ name, tone = Regexp.last_match(1).split(":", 2)
+ next shortcode if Emoji.custom?(name)
+
+ unicode = Emoji.lookup_unicode([Emoji.resolve_alias(name), tone].compact.join(":"))
+ unicode.presence || shortcode
+ end
+ end
+ end
+end
diff --git a/lib/markdown_endpoint/topic_list_renderer.rb b/lib/markdown_endpoint/topic_list_renderer.rb
index 7eb05033f94..cd098a9bf0d 100644
--- a/lib/markdown_endpoint/topic_list_renderer.rb
+++ b/lib/markdown_endpoint/topic_list_renderer.rb
@@ -60,7 +60,7 @@ module MarkdownEndpoint
end
def render_topic(topic)
- lines = ["## [#{escape_text(topic.title)}](#{topic.url})"]
+ lines = ["## [#{escape_text(EmojiConverter.convert(topic.title))}](#{topic.url})"]
lines.concat(["", '<div class="topic-metadata">', ""])
metadata = []
if topic.user
diff --git a/lib/markdown_endpoint/topic_renderer.rb b/lib/markdown_endpoint/topic_renderer.rb
index 217483a3902..a236b750b7b 100644
--- a/lib/markdown_endpoint/topic_renderer.rb
+++ b/lib/markdown_endpoint/topic_renderer.rb
@@ -59,7 +59,7 @@ module MarkdownEndpoint
def header(visible_post_count)
title = ContentLocalization.translated_topic_title(@topic, @guardian) || @topic.title
- lines = ["# #{escape_text(title)}", "", "**URL:** #{@topic.url}"]
+ lines = ["# #{escape_text(EmojiConverter.convert(title))}", "", "**URL:** #{@topic.url}"]
lines << "**Category:** #{escape_text(@topic.category.name)}" if @topic.category
tags = @topic_view.visible_tags.map(&:name)
lines << "**Tags:** #{tags.map { |tag| escape_text(tag) }.join(", ")}" if tags.present?
@@ -85,23 +85,24 @@ module MarkdownEndpoint
end
lines << "#### #{I18n.t("markdown_endpoints.post_date", timestamp: @timestamp.render(post.created_at, url: post.full_url))}"
lines.concat(["", "</div>", ""])
- lines << cached_body(cooked)
+ lines << cached_body(cooked, post_url: post.full_url)
lines.join("\n")
end
- def cached_body(cooked)
+ def cached_body(cooked, post_url:)
context = [
CookedProcessor::VERSION,
Discourse.base_url,
Discourse.asset_host,
I18n.locale,
+ post_url,
cooked,
].join("\0")
digest = Digest::SHA256.hexdigest(context)
Discourse
.cache
.fetch("markdown-endpoint:cooked:#{digest}", expires_in: 1.week) do
- CookedProcessor.to_markdown(cooked)
+ CookedProcessor.to_markdown(cooked, post_url:)
end
end
diff --git a/spec/lib/markdown_endpoint/cooked_processor_spec.rb b/spec/lib/markdown_endpoint/cooked_processor_spec.rb
index 8e5ced38cad..ce1eb7480cd 100644
--- a/spec/lib/markdown_endpoint/cooked_processor_spec.rb
+++ b/spec/lib/markdown_endpoint/cooked_processor_spec.rb
@@ -18,9 +18,64 @@ RSpec.describe MarkdownEndpoint::CookedProcessor do
markdown = described_class.to_markdown(html)
- expect(markdown).to include("## Heading", "**world**", "@sam", "#ruby", ":wave:")
+ expect(markdown).to include("## Heading", "**world**", "@sam", "#ruby", "πŸ‘‹")
expect(markdown).to include("| Name | Value |", 'puts "ok"')
- expect(markdown).to include("> **More**", "_Poll: Choose one (view on site)_")
+ expect(markdown).to include("> **More**", "_Poll: Choose one_")
+ end
+
+ it "converts standard emoji, aliases, and skin tones to Unicode" do
+ html = PrettyText.cook(":smile: :+1: :wave:t4: :thumbsup:t6:")
+
+ expect(described_class.to_markdown(html)).to eq("πŸ˜„ πŸ‘ πŸ‘‹πŸ½ πŸ‘πŸΏ")
+ end
+
+ it "preserves custom and unknown emoji shortcodes" do
+ html = <<~HTML
+ <p><img class="emoji emoji-custom" title=":custom_emoji:" src="/custom.png">
+ <img class="emoji" alt=":unknown_emoji:" src="/emoji.png"></p>
+ HTML
+
+ expect(described_class.to_markdown(html)).to eq(":custom_emoji::unknown_emoji:")
+ end
+
+ it "preserves literal shortcodes in code and URLs" do
+ html = <<~HTML
+ <p><code>:smile:</code> <a href="https://example.com/:wave:">Example</a></p>
+ <pre><code>:wave:t4:</code></pre>
+ HTML
+
+ expect(described_class.to_markdown(html)).to eq(
+ "`:smile:` [Example](https://example.com/:wave:)\n\n```\n:wave:t4:\n```",
+ )
+ end
+
+ it "keeps footnote markers and formatted definitions without dead fragment links" do
+ html = <<~HTML
+ <p>Some text<sup class="footnote-ref"><a href="#footnote-123-1" id="footnote-ref-123-1">[1]</a></sup>.</p>
+ <p>Repeated<sup class="footnote-ref"><a href="#footnote-123-1" id="footnote-ref-123-1:1">[1:1]</a></sup>.</p>
+ <section class="footnotes"><ol class="footnotes-list">
+ <li id="footnote-123-1" class="footnote-item"><p>A <strong>formatted</strong> note with a <a href="https://example.com">source</a> <a href="#footnote-ref-123-1" class="footnote-backref">β†©οΈŽ</a> <a href="#footnote-ref-123-1:1" class="footnote-backref">β†©οΈŽ</a></p></li>
+ </ol></section>
+ HTML
+
+ markdown = described_class.to_markdown(html)
+
+ expect(markdown).to include("Some text\\[1\\].", "Repeated\\[1:1\\].")
+ expect(markdown).to include("1. A **formatted** note with a [source](https://example.com)")
+ expect(markdown).not_to match(/\]\(#footnote|β†©οΈŽ/)
+ end
+
+ it "links named and unnamed polls, including nested polls, to their post" do
+ html = <<~HTML
+ <div class="poll" data-poll-title="Choose [one]"></div>
+ <details><summary>More</summary><div class="poll"></div></details>
+ HTML
+ post_url = "#{Discourse.base_url}/t/topic/123/2"
+
+ markdown = described_class.to_markdown(html, post_url:)
+
+ expect(markdown).to include("_Poll: Choose \\[one\\] ([view on site](#{post_url}))_")
+ expect(markdown).to include("> _Poll ([view on site](#{post_url}))_")
end
it "preserves quotes, oneboxes, lightboxes, and nested formatting" do
diff --git a/spec/lib/markdown_endpoint/emoji_converter_spec.rb b/spec/lib/markdown_endpoint/emoji_converter_spec.rb
new file mode 100644
index 00000000000..fdededcbf95
--- /dev/null
+++ b/spec/lib/markdown_endpoint/emoji_converter_spec.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe MarkdownEndpoint::EmojiConverter do
+ describe ".convert" do
+ before { Plugin::CustomEmoji.clear_cache }
+ after { Plugin::CustomEmoji.clear_cache }
+
+ it "preserves custom emoji that share standard names or aliases" do
+ Plugin::CustomEmoji.register("+1", "/custom/thumb.png")
+ Plugin::CustomEmoji.register("wave", "/custom/wave.png")
+
+ expect(described_class.convert(":+1: :wave: :wave:t4: :smile:")).to eq(
+ ":+1: :wave: :wave:t4: πŸ˜„",
+ )
+ end
+
+ it "preserves shortcodes when Unicode lookup returns a blank value" do
+ SiteSetting.emoji_deny_list = "peach"
+ Emoji.clear_cache
+
+ expect(described_class.convert(":peach: :unknown_emoji: :smile:")).to eq(
+ ":peach: :unknown_emoji: πŸ˜„",
+ )
+ end
+
+ it "converts standard emoji and toned aliases" do
+ expect(described_class.convert(":smile: :+1: :wave:t4: :thumbsup:t6:")).to eq("πŸ˜„ πŸ‘ πŸ‘‹πŸ½ πŸ‘πŸΏ")
+ end
+ end
+end
diff --git a/spec/requests/markdown_endpoints_spec.rb b/spec/requests/markdown_endpoints_spec.rb
index e3a5ff0d3f9..38b89c62f3b 100644
--- a/spec/requests/markdown_endpoints_spec.rb
+++ b/spec/requests/markdown_endpoints_spec.rb
@@ -32,6 +32,21 @@ RSpec.describe "Markdown endpoints" do
response.headers.fetch("Vary", "").split(",").map(&:strip)
end
+ it "converts emoji in topic headings and list links while preserving Markdown escaping" do
+ topic.update_columns(title: "Emoji [demo] :smile: :wave:t4: :thumbsup:t6: :custom_emoji:")
+ title = "Emoji \\[demo\\] πŸ˜„ πŸ‘‹πŸ½ πŸ‘πŸΏ :custom\\_emoji:"
+
+ get "/t/#{topic.slug}/#{topic.id}.md"
+
+ expect(response).to have_http_status(:ok)
+ expect(response.body).to start_with("# #{title}\n")
+
+ get "/latest.md"
+
+ expect(response).to have_http_status(:ok)
+ expect(response.body).to include("## [#{title}](#{topic.url})")
+ end
+
it "serves latest topics for Markdown requests to the homepage" do
get "/", headers: { "ACCEPT" => "text/markdown" }
@@ -398,6 +413,20 @@ RSpec.describe "Markdown endpoints" do
end
end
+ it "links identical polls to their own posts when caching Markdown bodies" do
+ cooked = '<div class="poll"></div>'
+ post.update_columns(cooked:)
+ reply = Fabricate(:post, topic:, user:, cooked:)
+
+ get "/t/#{topic.slug}/#{topic.id}.md"
+
+ expect(response).to have_http_status(:ok)
+ expect(response.body).to include(
+ "_Poll ([view on site](#{post.full_url}))_",
+ "_Poll ([view on site](#{reply.full_url}))_",
+ )
+ end
+
it "invalidates transformed bodies after cooked-only changes in the same second" do
get "/t/#{topic.slug}/#{topic.id}.md"
expect(response.body).to include(post.raw)