A platform for community discussion. Free, open, simple.
FEATURE: Improve emoji, footnotes, and polls in Markdown output (#43920)
Penar Musaraje6528f198926
diff --git a/config/initializers/markdown_endpoints.rb b/config/initializers/markdown_endpoints.rbindex e0428bc2633..e0e4a316c83 100644--- a/config/initializers/markdown_endpoints.rb+++ b/config/initializers/markdown_endpoints.rb@@ -4,6 +4,7 @@ require "reverse_markdown"require "markdown_endpoint/accept_header"require "markdown_endpoint/request_constraint"require "markdown_endpoint/vary_middleware"+require "markdown_endpoint/emoji_converter"require "markdown_endpoint/cooked_processor"require "markdown_endpoint/topic_renderer"require "markdown_endpoint/topic_list_renderer"diff --git a/lib/markdown_endpoint/cooked_processor.rb b/lib/markdown_endpoint/cooked_processor.rbindex 82a8ec2ba01..383505eb36e 100644--- a/lib/markdown_endpoint/cooked_processor.rb+++ b/lib/markdown_endpoint/cooked_processor.rb@@ -4,7 +4,7 @@ module MarkdownEndpointclass CookedProcessorBLOCK_TAG = "discourse-markdown-block"INLINE_TAG = "discourse-markdown-inline"- VERSION = 2+ VERSION = 4class PreservedBlockConverter < ReverseMarkdown::Converters::Basedef convert(node, _state = {})@@ -21,12 +21,13 @@ module MarkdownEndpointReverseMarkdown::Converters.register(BLOCK_TAG.to_sym, PreservedBlockConverter.new)ReverseMarkdown::Converters.register(INLINE_TAG.to_sym, PreservedInlineConverter.new)- def self.to_markdown(cooked_html)- new(cooked_html).to_markdown+ def self.to_markdown(cooked_html, post_url: nil)+ new(cooked_html, post_url:).to_markdownend- def initialize(cooked_html)+ def initialize(cooked_html, post_url: nil)@fragment = Nokogiri::HTML5.fragment(cooked_html.to_s)+ @post_url = post_urlenddef to_markdown@@ -36,6 +37,7 @@ module MarkdownEndpointreplace_hashtagsreplace_lightboxesreplace_code_blocks+ replace_footnotesreplace_quotesreplace_oneboxesreplace_details@@ -54,7 +56,10 @@ module MarkdownEndpointdef replace_emojis@fragment.css("img.emoji")- .each { |image| image.replace(preserved_inline(image["title"] || image["alt"] || "")) }+ .each do |image|+ shortcode = image["title"] || image["alt"] || ""+ image.replace(preserved_inline(EmojiConverter.convert(shortcode)))+ endenddef replace_mentions@@ -108,6 +113,13 @@ module MarkdownEndpointendend+ def replace_footnotes+ @fragment.css("a.footnote-backref").each(&:remove)+ @fragment+ .css("sup.footnote-ref a")+ .each { |anchor| anchor.replace(preserved_inline(escape_text(anchor.text))) }+ end+def replace_quotes@fragment.css("aside.quote")@@ -172,8 +184,10 @@ module MarkdownEndpoint.each do |poll|title =poll["data-poll-title"].presence ||- poll.at_css("[data-poll-title]")&.[]("data-poll-title").presence || "Poll"- poll.replace(preserved_block("_Poll: #{escape_text(title)} (view on site)_"))+ poll.at_css("[data-poll-title]")&.[]("data-poll-title").presence+ label = title.present? ? "Poll: #{escape_text(title)}" : "Poll"+ label += " ([view on site](#{absolute_url(@post_url)}))" if @post_url.present?+ poll.replace(preserved_block("_#{label}_"))endend@@ -196,7 +210,7 @@ module MarkdownEndpointenddef convert_html(html)- fragment = self.class.new(html)+ fragment = self.class.new(html, post_url: @post_url)fragment.to_markdownenddiff --git a/lib/markdown_endpoint/emoji_converter.rb b/lib/markdown_endpoint/emoji_converter.rbnew file mode 100644index 00000000000..de3b9f1f03c--- /dev/null+++ b/lib/markdown_endpoint/emoji_converter.rb@@ -0,0 +1,17 @@+# frozen_string_literal: true++module MarkdownEndpoint+ class EmojiConverter+ def self.convert(text)+ text+ .to_s+ .gsub(Emoji::EMOJI_CODE_REGEXP) do |shortcode|+ name, tone = Regexp.last_match(1).split(":", 2)+ next shortcode if Emoji.custom?(name)++ unicode = Emoji.lookup_unicode([Emoji.resolve_alias(name), tone].compact.join(":"))+ unicode.presence || shortcode+ end+ end+ end+enddiff --git a/lib/markdown_endpoint/topic_list_renderer.rb b/lib/markdown_endpoint/topic_list_renderer.rbindex 7eb05033f94..cd098a9bf0d 100644--- a/lib/markdown_endpoint/topic_list_renderer.rb+++ b/lib/markdown_endpoint/topic_list_renderer.rb@@ -60,7 +60,7 @@ module MarkdownEndpointenddef render_topic(topic)- lines = ["## [#{escape_text(topic.title)}](#{topic.url})"]+ lines = ["## [#{escape_text(EmojiConverter.convert(topic.title))}](#{topic.url})"]lines.concat(["", '<div class="topic-metadata">', ""])metadata = []if topic.userdiff --git a/lib/markdown_endpoint/topic_renderer.rb b/lib/markdown_endpoint/topic_renderer.rbindex 217483a3902..a236b750b7b 100644--- a/lib/markdown_endpoint/topic_renderer.rb+++ b/lib/markdown_endpoint/topic_renderer.rb@@ -59,7 +59,7 @@ module MarkdownEndpointdef header(visible_post_count)title = ContentLocalization.translated_topic_title(@topic, @guardian) || @topic.title- lines = ["# #{escape_text(title)}", "", "**URL:** #{@topic.url}"]+ lines = ["# #{escape_text(EmojiConverter.convert(title))}", "", "**URL:** #{@topic.url}"]lines << "**Category:** #{escape_text(@topic.category.name)}" if @topic.categorytags = @topic_view.visible_tags.map(&:name)lines << "**Tags:** #{tags.map { |tag| escape_text(tag) }.join(", ")}" if tags.present?@@ -85,23 +85,24 @@ module MarkdownEndpointendlines << "#### #{I18n.t("markdown_endpoints.post_date", timestamp: @timestamp.render(post.created_at, url: post.full_url))}"lines.concat(["", "</div>", ""])- lines << cached_body(cooked)+ lines << cached_body(cooked, post_url: post.full_url)lines.join("\n")end- def cached_body(cooked)+ def cached_body(cooked, post_url:)context = [CookedProcessor::VERSION,Discourse.base_url,Discourse.asset_host,I18n.locale,+ post_url,cooked,].join("\0")digest = Digest::SHA256.hexdigest(context)Discourse.cache.fetch("markdown-endpoint:cooked:#{digest}", expires_in: 1.week) do- CookedProcessor.to_markdown(cooked)+ CookedProcessor.to_markdown(cooked, post_url:)endenddiff --git a/spec/lib/markdown_endpoint/cooked_processor_spec.rb b/spec/lib/markdown_endpoint/cooked_processor_spec.rbindex 8e5ced38cad..ce1eb7480cd 100644--- a/spec/lib/markdown_endpoint/cooked_processor_spec.rb+++ b/spec/lib/markdown_endpoint/cooked_processor_spec.rb@@ -18,9 +18,64 @@ RSpec.describe MarkdownEndpoint::CookedProcessor domarkdown = described_class.to_markdown(html)- expect(markdown).to include("## Heading", "**world**", "@sam", "#ruby", ":wave:")+ expect(markdown).to include("## Heading", "**world**", "@sam", "#ruby", "π")expect(markdown).to include("| Name | Value |", 'puts "ok"')- expect(markdown).to include("> **More**", "_Poll: Choose one (view on site)_")+ expect(markdown).to include("> **More**", "_Poll: Choose one_")+ end++ it "converts standard emoji, aliases, and skin tones to Unicode" do+ html = PrettyText.cook(":smile: :+1: :wave:t4: :thumbsup:t6:")++ expect(described_class.to_markdown(html)).to eq("π π ππ½ ππΏ")+ end++ it "preserves custom and unknown emoji shortcodes" do+ html = <<~HTML+ <p><img class="emoji emoji-custom" title=":custom_emoji:" src="/custom.png">+ <img class="emoji" alt=":unknown_emoji:" src="/emoji.png"></p>+ HTML++ expect(described_class.to_markdown(html)).to eq(":custom_emoji::unknown_emoji:")+ end++ it "preserves literal shortcodes in code and URLs" do+ html = <<~HTML+ <p><code>:smile:</code> <a href="https://example.com/:wave:">Example</a></p>+ <pre><code>:wave:t4:</code></pre>+ HTML++ expect(described_class.to_markdown(html)).to eq(+ "`:smile:` [Example](https://example.com/:wave:)\n\n```\n:wave:t4:\n```",+ )+ end++ it "keeps footnote markers and formatted definitions without dead fragment links" do+ html = <<~HTML+ <p>Some text<sup class="footnote-ref"><a href="#footnote-123-1" id="footnote-ref-123-1">[1]</a></sup>.</p>+ <p>Repeated<sup class="footnote-ref"><a href="#footnote-123-1" id="footnote-ref-123-1:1">[1:1]</a></sup>.</p>+ <section class="footnotes"><ol class="footnotes-list">+ <li id="footnote-123-1" class="footnote-item"><p>A <strong>formatted</strong> note with a <a href="https://example.com">source</a> <a href="#footnote-ref-123-1" class="footnote-backref">β©οΈ</a> <a href="#footnote-ref-123-1:1" class="footnote-backref">β©οΈ</a></p></li>+ </ol></section>+ HTML++ markdown = described_class.to_markdown(html)++ expect(markdown).to include("Some text\\[1\\].", "Repeated\\[1:1\\].")+ expect(markdown).to include("1. A **formatted** note with a [source](https://example.com)")+ expect(markdown).not_to match(/\]\(#footnote|β©οΈ/)+ end++ it "links named and unnamed polls, including nested polls, to their post" do+ html = <<~HTML+ <div class="poll" data-poll-title="Choose [one]"></div>+ <details><summary>More</summary><div class="poll"></div></details>+ HTML+ post_url = "#{Discourse.base_url}/t/topic/123/2"++ markdown = described_class.to_markdown(html, post_url:)++ expect(markdown).to include("_Poll: Choose \\[one\\] ([view on site](#{post_url}))_")+ expect(markdown).to include("> _Poll ([view on site](#{post_url}))_")endit "preserves quotes, oneboxes, lightboxes, and nested formatting" dodiff --git a/spec/lib/markdown_endpoint/emoji_converter_spec.rb b/spec/lib/markdown_endpoint/emoji_converter_spec.rbnew file mode 100644index 00000000000..fdededcbf95--- /dev/null+++ b/spec/lib/markdown_endpoint/emoji_converter_spec.rb@@ -0,0 +1,32 @@+# frozen_string_literal: true++require "rails_helper"++RSpec.describe MarkdownEndpoint::EmojiConverter do+ describe ".convert" do+ before { Plugin::CustomEmoji.clear_cache }+ after { Plugin::CustomEmoji.clear_cache }++ it "preserves custom emoji that share standard names or aliases" do+ Plugin::CustomEmoji.register("+1", "/custom/thumb.png")+ Plugin::CustomEmoji.register("wave", "/custom/wave.png")++ expect(described_class.convert(":+1: :wave: :wave:t4: :smile:")).to eq(+ ":+1: :wave: :wave:t4: π",+ )+ end++ it "preserves shortcodes when Unicode lookup returns a blank value" do+ SiteSetting.emoji_deny_list = "peach"+ Emoji.clear_cache++ expect(described_class.convert(":peach: :unknown_emoji: :smile:")).to eq(+ ":peach: :unknown_emoji: π",+ )+ end++ it "converts standard emoji and toned aliases" do+ expect(described_class.convert(":smile: :+1: :wave:t4: :thumbsup:t6:")).to eq("π π ππ½ ππΏ")+ end+ end+enddiff --git a/spec/requests/markdown_endpoints_spec.rb b/spec/requests/markdown_endpoints_spec.rbindex e3a5ff0d3f9..38b89c62f3b 100644--- a/spec/requests/markdown_endpoints_spec.rb+++ b/spec/requests/markdown_endpoints_spec.rb@@ -32,6 +32,21 @@ RSpec.describe "Markdown endpoints" doresponse.headers.fetch("Vary", "").split(",").map(&:strip)end+ it "converts emoji in topic headings and list links while preserving Markdown escaping" do+ topic.update_columns(title: "Emoji [demo] :smile: :wave:t4: :thumbsup:t6: :custom_emoji:")+ title = "Emoji \\[demo\\] π ππ½ ππΏ :custom\\_emoji:"++ get "/t/#{topic.slug}/#{topic.id}.md"++ expect(response).to have_http_status(:ok)+ expect(response.body).to start_with("# #{title}\n")++ get "/latest.md"++ expect(response).to have_http_status(:ok)+ expect(response.body).to include("## [#{title}](#{topic.url})")+ end+it "serves latest topics for Markdown requests to the homepage" doget "/", headers: { "ACCEPT" => "text/markdown" }@@ -398,6 +413,20 @@ RSpec.describe "Markdown endpoints" doendend+ it "links identical polls to their own posts when caching Markdown bodies" do+ cooked = '<div class="poll"></div>'+ post.update_columns(cooked:)+ reply = Fabricate(:post, topic:, user:, cooked:)++ get "/t/#{topic.slug}/#{topic.id}.md"++ expect(response).to have_http_status(:ok)+ expect(response.body).to include(+ "_Poll ([view on site](#{post.full_url}))_",+ "_Poll ([view on site](#{reply.full_url}))_",+ )+ end+it "invalidates transformed bodies after cooked-only changes in the same second" doget "/t/#{topic.slug}/#{topic.id}.md"expect(response.body).to include(post.raw)