A platform for community discussion. Free, open, simple.
DEV: Simplify `PrettyText` email formatting helpers and their specs
Régis Hanolea4aecc02e35
diff --git a/lib/pretty_text.rb b/lib/pretty_text.rbindex e163234ea5d..9a1043e7f3e 100644--- a/lib/pretty_text.rb+++ b/lib/pretty_text.rb@@ -510,21 +510,14 @@ module PrettyTextdoc.css("a[href]").each do |a|- href = a["href"].to_s- next if href.blank?- next if href.start_with?("mailto:")- next if href.start_with?(Discourse.base_url)- next if URI(href).host.present?-- a["href"] = (- if href.start_with?(Discourse.base_path)- "#{Discourse.base_url_no_prefix}#{href}"- else- "#{Discourse.base_url}#{href}"- end- )+ href = a["href"]+ next if href.blank? || href.start_with?("mailto:") || URI(href).host.present?++ prefix =+ href.start_with?(Discourse.base_path) ? Discourse.base_url_no_prefix : Discourse.base_url+ a["href"] = "#{prefix}#{href}"rescue URI::Error- # leave it+ nilendend@@ -542,17 +535,15 @@ module PrettyText.css("iframe[src*='player.vimeo.com']").each do |iframe|vimeo_url = vimeo_url_from_iframe(iframe)- next if vimeo_url.blank?-- paragraph = doc.document.create_element("p")- link = doc.document.create_element("a")- link["href"] = vimeo_url- link.content = vimeo_url- paragraph.add_child(link)- iframe.replace(paragraph)+ iframe.replace(link_paragraph(doc, vimeo_url, vimeo_url)) if vimeo_urlendend+ def self.link_paragraph(doc, href, text)+ document = doc.document+ document.create_element("p") { |p| p << document.create_element("a", text, href: href) }+ end+def self.vimeo_url_from_iframe(iframe)if iframe["data-original-href"].present?url = UrlHelper.normalized_encode(iframe["data-original-href"])@@ -563,15 +554,10 @@ module PrettyTexturi = UrlHelper.relaxed_parse(iframe["src"].to_s)return nil if uri.nil? || uri.host&.downcase != VIMEO_PLAYER_HOST- video_id = VIMEO_PLAYER_PATH.match(uri.path.to_s)&.[](:id)+ video_id = uri.path.to_s[VIMEO_PLAYER_PATH, :id]return nil if video_id.nil?- unlisted_hash = vimeo_unlisted_hash(uri.query)- if unlisted_hash- "https://vimeo.com/#{video_id}/#{unlisted_hash}"- else- "https://vimeo.com/#{video_id}"- end+ ["https://vimeo.com", video_id, vimeo_unlisted_hash(uri.query)].compact.join("/")enddef self.vimeo_unlisted_hash(query)@@ -590,42 +576,13 @@ module PrettyText.each do |a|next if !Upload.secure_uploads_url?(a["href"])- non_image_media = %w[video audio].include?(a&.parent&.name)- target = non_image_media ? a.parent : a- if target.to_s.include?("stripped-secure-view-media") ||- target.to_s.include?("stripped-secure-view-upload")- next- end-- next if a.css("img[src]").empty? && !non_image_media-- if a.classes.include?("lightbox")- img = a.css("img[src]").first- srcset = img&.attributes&.[]("srcset")&.value- if srcset- # if available, use the first image from the srcset here- # so we get the optimized image instead of the possibly huge original- url = srcset.split(",").first- else- url = img["src"]- end- a.add_next_sibling secure_uploads_placeholder(- doc,- url,- width: img["width"],- height: img["height"],- )- a.remove- else- width = non_image_media ? nil : a.at_css("img").attr("width")- height = non_image_media ? nil : a.at_css("img").attr("height")- target.add_next_sibling secure_uploads_placeholder(- doc,- a["href"],- width: width,- height: height,- )- target.remove+ if %w[video audio].include?(a.parent&.name)+ a.parent.replace(secure_uploads_placeholder(doc, a["href"]))+ elsif (img = a.at_css("img[src]"))+ url = a.classes.include?("lightbox") ? optimized_src(img) : a["href"]+ a.replace(+ secure_uploads_placeholder(doc, url, width: img["width"], height: img["height"]),+ )endend@@ -633,50 +590,39 @@ module PrettyTextdoc.css("img[src]").each do |img|- url =- if img.parent.classes.include?("aspect-image") && img.attributes["srcset"].present?- # we are using the first image from the srcset here so we get the- # optimized image instead of the original, because an optimized- # image may be used for the onebox thumbnail- srcset = img.attributes["srcset"].value- srcset.split(",").first- else- img["src"]- end+ # an optimized image may be used for the onebox thumbnail+ url = img.parent.classes.include?("aspect-image") ? optimized_src(img) : img["src"]+ next if !Upload.secure_uploads_url?(url)width = img["width"]height = img["height"]onebox_type = nil- if img.ancestors(".onebox-body").any?- if img.classes.include?("onebox-avatar-inline")- onebox_type = "avatar-inline"- else- onebox_type = "thumbnail"- end- end-- # we always want this to be tiny and without any special stylesif img.classes.include?("site-icon")- onebox_type = nil- width = 16- height = 16+ # we always want this to be tiny and without any special styles+ width = height = 16+ elsif img.ancestors(".onebox-body").any?+ onebox_type = img.classes.include?("onebox-avatar-inline") ? "avatar-inline" : "thumbnail"end- if Upload.secure_uploads_url?(url)- img.add_next_sibling secure_uploads_placeholder(- doc,- url,- onebox_type: onebox_type,- width: width,- height: height,- )- img.remove- end+ img.replace(+ secure_uploads_placeholder(+ doc,+ url,+ onebox_type: onebox_type,+ width: width,+ height: height,+ ),+ )endend- def self.secure_uploads_placeholder(doc, url, onebox_type: false, width: nil, height: nil)+ # The first srcset candidate is optimized, unlike the possibly huge original in src+ def self.optimized_src(img)+ img["srcset"].present? ? img["srcset"].split(",").first : img["src"]+ end++ def self.secure_uploads_placeholder(doc, url, onebox_type: nil, width: nil, height: nil)notice = doc.document.create_element("div")notice["class"] = "secure-upload-notice"notice["data-stripped-secure-upload"] = url.to_sdiff --git a/spec/lib/pretty_text_spec.rb b/spec/lib/pretty_text_spec.rbindex 812796715f0..111661a9cb3 100644--- a/spec/lib/pretty_text_spec.rb+++ b/spec/lib/pretty_text_spec.rb@@ -1374,295 +1374,195 @@ RSpec.describe PrettyText doenddescribe "format_for_email" do- context "when (sub)domain" do- before { Discourse.stubs(:base_path).returns("") }-- it "does not crash" do- html = <<~HTML- <a href="mailto:[email protected]?subject=Your%20post%20at%20http://try.discourse.org/t/discussion-happens-so-much/127/1000?u=supermathie">test</a>- HTML-- expect(described_class.format_for_email(html, post)).to eq <<~HTML- <a href="mailto:[email protected]?subject=Your%20post%20at%20http://try.discourse.org/t/discussion-happens-so-much/127/1000?u=supermathie">test</a>- HTML- end-+ describe ".make_all_links_absolute" doit "adds base url to relative links" do- html = <<~HTML- <p><a class="mention" href="/u/wiseguy">@wiseguy</a>, <a class="mention" href="/u/trollol">@trollol</a> what do you guys think?</p>- HTML+ html = %(<a href="/u/wiseguy">@wiseguy</a>)- expect(described_class.format_for_email(html, post)).to eq <<~HTML- <p><a class="mention" href="#{Discourse.base_url}/u/wiseguy">@wiseguy</a>, <a class="mention" href="#{Discourse.base_url}/u/trollol">@trollol</a> what do you guys think?</p>- HTML+ expect(described_class.format_for_email(html, post)).to match_html(+ %(<a href="#{Discourse.base_url}/u/wiseguy">@wiseguy</a>),+ )end- it "doesn't change external absolute links" do+ it "leaves absolute, mailto and invalid links unchanged" dohtml = <<~HTML- <p>Check out <a href="http://mywebsite.com/users/boss">this guy</a>.</p>+ <a href="http://mywebsite.com/users/boss">external</a>+ <a href="#{Discourse.base_url}/users/boss">internal</a>+ <a href="mailto:[email protected]?subject=Your%20post%20at%20http://try.discourse.org/t/127/1000?u=supermathie">mailto</a>+ <a href="not a real url">invalid</a>HTMLexpect(described_class.format_for_email(html, post)).to eq(html)end- it "doesn't change internal absolute links" do- html = <<~HTML- <p>Check out <a href="#{Discourse.base_url}/users/boss">this guy</a>.</p>- HTML+ context "with a subfolder" do+ before { set_subfolder "/forum" }- expect(described_class.format_for_email(html, post)).to eq(html)- end+ it "adds base url to relative links, whether or not they include the subfolder" do+ html = %(<a href="/forum/u/wiseguy">a</a><a href="/u/trollol">b</a>)- it "can tolerate invalid URLs" do- html = <<~HTML- <p>Check out <a href="not a real url">this guy</a>.</p>- HTML-- expect(described_class.format_for_email(html, post)).to eq(html)+ expect(described_class.format_for_email(html, post)).to match_html(+ %(<a href="#{Discourse.base_url}/u/wiseguy">a</a><a href="#{Discourse.base_url}/u/trollol">b</a>),+ )+ endend+ end- it "doesn't change mailto" do- html = <<~HTML- <p>Contact me at <a href="mailto:[email protected]">this address</a>.</p>- HTML+ describe ".convert_vimeo_iframes" do+ it "converts a Vimeo iframe to a link, ignoring an invalid unlisted hash" do+ html = %(<p>Video:</p><iframe src="https://player.vimeo.com/video/1?h=not-valid!"></iframe>)- expect(described_class.format_for_email(html, post)).to eq(html)+ expect(described_class.format_for_email(html, post)).to match_html(<<~HTML)+ <p>Video:</p>+ <p><a href="https://vimeo.com/1">https://vimeo.com/1</a></p>+ HTMLend- it "prefers data-original-href attribute to get Vimeo iframe link and escapes it" do- html = <<~HTML- <p>Check out this video – <iframe src='https://player.vimeo.com/video/329875646' data-original-href='https://vimeo.com/329875646/> <script>alert(1)</script>'></iframe>.</p>- HTML+ it "converts an unlisted Vimeo embed using its hash" do+ html = %(<iframe src="https://player.vimeo.com/video/508864124?h=fcbbcc92fa"></iframe>)- expect(described_class.format_for_email(html, post)).to match(- Regexp.escape("https://vimeo.com/329875646/%3E%20%3Cscript%3Ealert(1)%3C/script%3E"),- )+ expect(described_class.format_for_email(html, post)).to match_html(<<~HTML)+ <p><a href="https://vimeo.com/508864124/fcbbcc92fa">https://vimeo.com/508864124/fcbbcc92fa</a></p>+ HTMLend- it "creates a valid URL when data-original-href is missing from Vimeo link" do- html = <<~HTML- <iframe src="https://player.vimeo.com/video/508864124?h=fcbbcc92fa" width="640" height="360" frameborder="0" allow="autoplay; fullscreen; picture-in-picture" allowfullscreen></iframe>- HTML+ it "prefers data-original-href to get the Vimeo link and escapes it" do+ html =+ "<iframe src='https://player.vimeo.com/video/329875646' data-original-href='https://vimeo.com/329875646/> <script>alert(1)</script>'></iframe>"- expect(described_class.format_for_email(html, post)).to match(- "https://vimeo.com/508864124/fcbbcc92fa",+ expect(described_class.format_for_email(html, post)).to include(+ "https://vimeo.com/329875646/%3E%20%3Cscript%3Ealert(1)%3C/script%3E",)end- describe "#convert_vimeo_iframes" do- it "converts <iframe> to <a>" do- html = <<~HTML- <p>This is a Vimeo link:</p>- <iframe width="640" height="360" src="https://player.vimeo.com/video/1" data-original-href="https://vimeo.com/1" frameborder="0" allowfullscreen="" seamless="seamless" sandbox="allow-same-origin allow-scripts allow-forms allow-popups allow-popups-to-escape-sandbox allow-presentation"></iframe>- HTML+ it "leaves a malformed Vimeo source without rendering it as markup" do+ iframe_source = "https://player.vimeo.com/video/123'></a><img src=x onerror=alert(1)>"+ html = %(<iframe src="#{iframe_source}"></iframe>)- md = described_class.format_for_email(html, post)+ fragment = Nokogiri::HTML5.fragment(described_class.format_for_email(html, post))- expect(md).not_to include("<iframe")- expect(md).to match_html(<<~HTML)- <p>This is a Vimeo link:</p>- <p><a href="https://vimeo.com/1">https://vimeo.com/1</a></p>- HTML- end-- it "leaves a malformed Vimeo source without rendering it as markup" do- iframe_source = "https://player.vimeo.com/video/123'></a><img src=x onerror=alert(1)>"- html = %(<iframe src="#{iframe_source}"></iframe>)-- fragment = Nokogiri::HTML5.fragment(described_class.format_for_email(html, post))-- aggregate_failures do- expect(fragment.css("img")).to be_empty- expect(fragment.at_css("iframe")["src"]).to eq(iframe_source)- end- end-- it "leaves a non-Vimeo iframe containing the player domain unchanged" do- iframe_source =- "https://www.instagram.com/?x=player.vimeo.com/<img src=x onerror=alert(1)>"- html = %(<iframe src="#{iframe_source}"></iframe>)-- fragment = Nokogiri::HTML5.fragment(described_class.format_for_email(html, post))-- aggregate_failures do- expect(fragment.css("a")).to be_empty- expect(fragment.at_css("iframe")["src"]).to eq(iframe_source)- end- end-- it "converts an unlisted Vimeo embed using its hash" do- html = %(<iframe src="https://player.vimeo.com/video/508864124?h=fcbbcc92fa"></iframe>)-- expect(described_class.format_for_email(html, post)).to match_html(<<~HTML)- <p><a href="https://vimeo.com/508864124/fcbbcc92fa">https://vimeo.com/508864124/fcbbcc92fa</a></p>- HTML- end-- it "leaves an iframe with a non-http data-original-href unchanged" do- html =- %(<iframe src="https://player.vimeo.com/video/1" data-original-href="javascript:alert(1)"></iframe>)-- fragment = Nokogiri::HTML5.fragment(described_class.format_for_email(html, post))-- aggregate_failures do- expect(fragment.css("a")).to be_empty- expect(fragment.at_css("iframe")["data-original-href"]).to eq("javascript:alert(1)")- end+ aggregate_failures do+ expect(fragment.css("img")).to be_empty+ expect(fragment.at_css("iframe")["src"]).to eq(iframe_source)endend- describe "#strip_secure_uploads" do- before do- setup_s3- SiteSetting.s3_cdn_url = "https://s3.cdn.com"- SiteSetting.secure_uploads = true- SiteSetting.login_required = true- end+ it "leaves a non-Vimeo iframe containing the player domain unchanged" do+ iframe_source = "https://www.instagram.com/?x=player.vimeo.com/<img src=x onerror=alert(1)>"+ html = %(<iframe src="#{iframe_source}"></iframe>)- it "replaces secure video content" do- html = <<~HTML- <video width="100%" height="100%" controls="">- <source src="#{Discourse.base_url}/secure-uploads/original/1X/some-video.mp4">- <a href="#{Discourse.base_url}/secure-uploads/original/1X/some-video.mp4">Video label</a>- </source>- </video>- HTML-- md = described_class.format_for_email(html, post)-- expect(md).not_to include("<video")- expect(md.to_s).to match(I18n.t("emails.secure_uploads_placeholder"))- expect(md.to_s).not_to match(SiteSetting.Upload.s3_cdn_url)- end-- it "replaces secure audio content" do- html = <<~HTML- <audio controls>- <source src="#{Discourse.base_url}/secure-uploads/original/1X/some-audio.mp3">- <a href="#{Discourse.base_url}/secure-uploads/original/1X/some-audio.mp3">Audio label</a>- </source>- </audio>- HTML-- md = described_class.format_for_email(html, post)+ fragment = Nokogiri::HTML5.fragment(described_class.format_for_email(html, post))- expect(md).not_to include("<audio")- expect(md.to_s).to match(I18n.t("emails.secure_uploads_placeholder"))- expect(md.to_s).not_to match(SiteSetting.Upload.s3_cdn_url)+ aggregate_failures do+ expect(fragment.css("a")).to be_empty+ expect(fragment.at_css("iframe")["src"]).to eq(iframe_source)end+ end- it "replaces secure uploads within a link with a placeholder, keeping the url in an attribute" do- url = "#{Discourse.base_url}\/secure-uploads/original/1X/testimage.png"- html = <<~HTML- <a href="#{url}"><img src="/secure-uploads/original/1X/testimage.png"></a>- HTML+ it "leaves an iframe with a non-http data-original-href unchanged" do+ html =+ %(<iframe src="https://player.vimeo.com/video/1" data-original-href="javascript:alert(1)"></iframe>)- md = described_class.format_for_email(html, post)+ fragment = Nokogiri::HTML5.fragment(described_class.format_for_email(html, post))- expect(md).not_to include("<img")- expect(md).to include("Redacted")- expect(md).to include("data-stripped-secure-upload=\"#{url}\"")+ aggregate_failures do+ expect(fragment.css("a")).to be_empty+ expect(fragment.at_css("iframe")["data-original-href"]).to eq("javascript:alert(1)")end+ end+ end- it "does not create nested redactions from double processing because of the view media link" do- url = "#{Discourse.base_url}\/secure-uploads/original/1X/testimage.png"- html = <<~HTML- <a href="#{url}"><img src="/secure-uploads/original/1X/testimage.png"></a>- HTML-- md = described_class.format_for_email(html, post)-- expect(md.scan(/stripped-secure-view-upload/).length).to eq(1)- expect(md.scan(/Redacted/).length).to eq(1)- end-- it "replaces secure images with a placeholder, keeping the url in an attribute" do- url = "/secure-uploads/original/1X/testimage.png"- html = <<~HTML- <img src="#{url}" width="20" height="20">- HTML-- md = described_class.format_for_email(html, post)-- expect(md).not_to include("<img")- expect(md).to include("Redacted")- expect(md).to include("data-stripped-secure-upload=\"#{url}\"")- expect(md).to include("data-width=\"20\"")- expect(md).to include("data-height=\"20\"")- end+ describe ".strip_secure_uploads" do+ before do+ setup_s3+ SiteSetting.secure_uploads = true+ SiteSetting.login_required = true+ end- it "only marks images inside a onebox as onebox images" do- html = <<~HTML- <aside class="onebox"><div class="onebox-body"><img src="/secure-uploads/original/1X/a.png" class="onebox-avatar-inline"></div></aside>- <p><img src="/secure-uploads/original/1X/b.png"></p>- HTML+ it "replaces secure video and audio content" do+ html = <<~HTML+ <video controls>+ <source src="#{Discourse.base_url}/secure-uploads/original/1X/some-video.mp4">+ <a href="#{Discourse.base_url}/secure-uploads/original/1X/some-video.mp4">Video label</a>+ </video>+ <audio controls>+ <source src="#{Discourse.base_url}/secure-uploads/original/1X/some-audio.mp3">+ <a href="#{Discourse.base_url}/secure-uploads/original/1X/some-audio.mp3">Audio label</a>+ </audio>+ HTML- notices =- Nokogiri::HTML5.fragment(described_class.format_for_email(html, post)).css(- ".secure-upload-notice",- )+ md = described_class.format_for_email(html, post)- expect(notices.map { |notice| notice["data-onebox-type"] }).to eq(["avatar-inline", nil])- end+ expect(md).not_to include("<video", "<audio")+ expect(md.scan(I18n.t("emails.secure_uploads_placeholder")).size).to eq(2)+ end- it "keeps a crafted secure upload URL as attribute data" do- url =- %(#{Discourse.base_url}/secure-uploads/original/1X/a"><img src=x onerror=alert(1)>b.png)+ it "replaces secure uploads within a link with a placeholder, idempotently" do+ url = "#{Discourse.base_url}/secure-uploads/original/1X/testimage.png"+ html = %(<a href="#{url}"><img src="/secure-uploads/original/1X/testimage.png"></a>)- fragment =- Nokogiri::HTML5.fragment(described_class.format_for_email(%(<img src='#{url}'>), post))- notice = fragment.at_css("div.secure-upload-notice")+ md = described_class.format_for_email(described_class.format_for_email(html, post), post)- aggregate_failures do- expect(fragment.css("img")).to be_empty- expect(notice["data-stripped-secure-upload"]).to eq(url)- end- end+ expect(md).not_to include("<img")+ expect(md).to include(%(data-stripped-secure-upload="#{url}"))+ expect(md.scan("stripped-secure-view-upload").size).to eq(1)+ end- it "discards non-numeric secure upload dimensions" do- html =- %(<img src="/secure-uploads/original/1X/testimage.png" width="20 onmouseover=alert(1) x" height="20">)+ it "replaces secure images with a placeholder, keeping the url in an attribute" do+ url = "/secure-uploads/original/1X/testimage.png"+ html = %(<img src="#{url}" width="20" height="20">)- notice =- Nokogiri::HTML5.fragment(described_class.format_for_email(html, post)).at_css(- "div.secure-upload-notice",- )+ md = described_class.format_for_email(html, post)- aggregate_failures do- expect(notice.attributes.keys).not_to include("onmouseover", "data-width")- expect(notice["data-height"]).to eq("20")- end- end+ expect(md).not_to include("<img")+ expect(md).to include(+ I18n.t("emails.secure_uploads_placeholder"),+ %(data-stripped-secure-upload="#{url}"),+ %(data-width="20"),+ %(data-height="20"),+ )end- end-- context "when subfolder" do- before { Discourse.stubs(:base_path).returns("/forum") }- it "adds base url to relative links" do+ it "only marks images inside a onebox as onebox images" dohtml = <<~HTML- <p><a class="mention" href="/forum/u/wiseguy">@wiseguy</a>, <a class="mention" href="/forum/u/trollol">@trollol</a> what do you guys think?</p>+ <aside class="onebox"><div class="onebox-body"><img src="/secure-uploads/original/1X/a.png" class="onebox-avatar-inline"></div></aside>+ <p><img src="/secure-uploads/original/1X/b.png"></p>HTML- expect(described_class.format_for_email(html, post)).to eq <<~HTML- <p><a class="mention" href="#{Discourse.base_url}/u/wiseguy">@wiseguy</a>, <a class="mention" href="#{Discourse.base_url}/u/trollol">@trollol</a> what do you guys think?</p>- HTML+ notices =+ Nokogiri::HTML5.fragment(described_class.format_for_email(html, post)).css(+ ".secure-upload-notice",+ )++ expect(notices.map { |notice| notice["data-onebox-type"] }).to eq(["avatar-inline", nil])end- it "doesn't change external absolute links" do- html = <<~HTML- <p>Check out <a href="https://mywebsite.com/users/boss">this guy</a>.</p>- HTML+ it "keeps a crafted secure upload URL as attribute data" do+ url =+ %(#{Discourse.base_url}/secure-uploads/original/1X/a"><img src=x onerror=alert(1)>b.png)- expect(described_class.format_for_email(html, post)).to eq(html)+ fragment =+ Nokogiri::HTML5.fragment(described_class.format_for_email(%(<img src='#{url}'>), post))+ notice = fragment.at_css("div.secure-upload-notice")++ aggregate_failures do+ expect(fragment.css("img")).to be_empty+ expect(notice["data-stripped-secure-upload"]).to eq(url)+ endend- it "doesn't change internal absolute links" do- html = <<~HTML- <p>Check out <a href="#{Discourse.base_url}/users/boss">this guy</a>.</p>- HTML+ it "discards non-numeric secure upload dimensions" do+ html =+ %(<img src="/secure-uploads/original/1X/testimage.png" width="20 onmouseover=alert(1) x" height="20">)- expect(described_class.format_for_email(html, post)).to eq(html)+ notice =+ Nokogiri::HTML5.fragment(described_class.format_for_email(html, post)).at_css(+ "div.secure-upload-notice",+ )++ aggregate_failures do+ expect(notice.attributes.keys).not_to include("onmouseover", "data-width")+ expect(notice["data-height"]).to eq("20")+ endendendend