A platform for community discussion. Free, open, simple.
DEV: Use the Content-Type charset when encoding is missing from the document metadata when parsing companion topic pages (#44695)
Juan David Martínez Cubillos306221a727ad
diff --git a/app/models/topic_embed.rb b/app/models/topic_embed.rbindex a3395e5dc27..bbd744da3a5 100644--- a/app/models/topic_embed.rb+++ b/app/models/topic_embed.rb@@ -192,17 +192,27 @@ class TopicEmbed < ActiveRecord::Basereturn if uri.blank?begin- html = FinalDestination::HTTP.get(uri)+ response = FinalDestination::HTTP.get_response(uri)rescue OpenURI::HTTPError, Net::OpenTimeout, FinalDestination::SSRFDetector::DisallowedIpErrorreturnend- parse_html(html, uri.to_s)+ parse_html(response.body, uri.to_s, content_type: response["content-type"])end- def self.parse_html(html, url)+ def self.parse_html(html, url, content_type: nil)require "ruby-readability"+ encoding = Encodings.charset_from_content_type(content_type)++ # A transport-level charset can misrepresent the document. When the response+ # header declares a non-Unicode charset but the page itself declares UTF-8+ # and the bytes are valid UTF-8, the page's declaration wins.+ if encoding.present? && !Encodings.unicode_encoding?(encoding) &&+ Encodings.charset_from_document(html) == "utf-8" && Encodings.valid_utf8?(html)+ encoding = "utf-8"+ end+opts = {tags: %w[div@@ -237,12 +247,19 @@ class TopicEmbed < ActiveRecord::Baseopts[:blacklist] = SiteSetting.blocked_embed_selectors if SiteSetting.blocked_embed_selectors.present?+ opts[:encoding] = encoding if encoding.present?allowed_embed_classnames =SiteSetting.allowed_embed_classnames if SiteSetting.allowed_embed_classnames.present?response = FetchResponse.new- raw_doc = Nokogiri.HTML5(html)+ raw_doc =+ begin+ Nokogiri.HTML5(html, encoding: encoding)+ rescue ArgumentError, EncodingError+ opts.delete(:encoding)+ Nokogiri.HTML5(html)+ endresponse.url = urldiff --git a/lib/encodings.rb b/lib/encodings.rbindex b7e42fd2584..83e797999af 100644--- a/lib/encodings.rb+++ b/lib/encodings.rb@@ -12,4 +12,48 @@ module Encodingsstring.presence&.delete_prefix!(BOM)stringend++ # Returns the lowercase charset name declared in an HTTP Content-Type header,+ # or nil when absent or unsupported. HTML parsers like Nokogiri only receive+ # the raw body and cannot see transport-level headers, so callers must pass+ # the extracted charset along explicitly.+ def self.charset_from_content_type(content_type)+ return if content_type.blank?++ charset = content_type[/\bcharset\s*=\s*"?([a-z0-9_-]+)"?/i, 1]+ return if charset.blank?++ encoding = Encoding.find(charset)+ return if encoding.dummy?++ # Ruby has no identity converter, so skip the probe for UTF-8 itself.+ Encoding::Converter.new(encoding, Encoding::UTF_8) if encoding != Encoding::UTF_8++ charset.downcase+ rescue ArgumentError, Encoding::ConverterNotFoundError+ nil+ end++ def self.charset_from_document(html)+ head = html.to_s.byteslice(0, 1024).to_s+ charset = head[/<meta[^>]*\bcharset\s*=\s*["']?([a-z0-9_-]+)/i, 1]+ charset&.downcase+ end++ def self.valid_utf8?(string)+ string.to_s.dup.force_encoding(Encoding::UTF_8).valid_encoding?+ end++ def self.unicode_encoding?(charset)+ encoding = Encoding.find(charset)+ [+ Encoding::UTF_8,+ Encoding::UTF_16LE,+ Encoding::UTF_16BE,+ Encoding::UTF_32LE,+ Encoding::UTF_32BE,+ ].include?(encoding)+ rescue ArgumentError+ false+ endenddiff --git a/lib/retrieve_title.rb b/lib/retrieve_title.rbindex 8ec16002d27..4309b198983 100644--- a/lib/retrieve_title.rb+++ b/lib/retrieve_title.rb@@ -27,8 +27,9 @@ module RetrieveTitledoc = nilbegindoc = Nokogiri.HTML5(html, encoding:)- rescue ArgumentError- # invalid HTML (Eg: too many attributes, status tree too deep) - ignore+ rescue ArgumentError, EncodingError+ # invalid HTML (Eg: too many attributes, status tree too deep) or an+ # unsupported encoding - ignore# Error in nokogumbo is not specialized, uses generic ArgumentError# see: https://www.rubydoc.info/gems/nokogiri/Nokogiri/HTML5#label-Error+reportingend@@ -102,12 +103,7 @@ module RetrieveTitlecurrent = chunkend- if !encoding && content_type = _response["content-type"]&.strip&.downcase- if content_type =~ /charset="?([a-z0-9_-]+)"?/- encoding = Regexp.last_match(1)- encoding = nil if !Encoding.list.map(&:name).map(&:downcase).include?(encoding)- end- end+ encoding ||= Encodings.charset_from_content_type(_response["content-type"])max_size = max_chunk_size(uri) * 1024title = extract_title(current, encoding)diff --git a/spec/lib/encodings_spec.rb b/spec/lib/encodings_spec.rbindex 3db2e416f3f..ddaca4500c5 100644--- a/spec/lib/encodings_spec.rb+++ b/spec/lib/encodings_spec.rb@@ -8,4 +8,93 @@ RSpec.describe Encodings doit "replaces invalid bytes" doexpect(Encodings.force_utf8("caf\xFFe")).to eq("cafe")end++ describe ".charset_from_content_type" do+ it "returns the lowercase charset from the header" do+ expect(Encodings.charset_from_content_type("text/html; charset=UTF-8")).to eq("utf-8")+ end++ it "returns the charset when quoted" do+ expect(Encodings.charset_from_content_type('text/html; charset="utf-8"')).to eq("utf-8")+ end++ it "returns the charset when no space precedes it" do+ expect(Encodings.charset_from_content_type("text/html;charset=euc-jp")).to eq("euc-jp")+ end++ it "returns the charset when whitespace surrounds the equals sign" do+ expect(Encodings.charset_from_content_type("text/html; charset = Shift_JIS")).to eq(+ "shift_jis",+ )+ end++ it "returns nil when no charset is declared" do+ expect(Encodings.charset_from_content_type("text/html")).to be_nil+ end++ it "returns nil when the charset is unsupported" do+ expect(Encodings.charset_from_content_type("text/html; charset=not-an-encoding")).to be_nil+ end++ it "returns nil for dummy encodings without a UTF-8 converter" do+ expect(Encodings.charset_from_content_type("text/html; charset=UTF-7")).to be_nil+ expect(Encodings.charset_from_content_type("text/html; charset=UTF-16")).to be_nil+ end++ it "returns nil when the header is blank" do+ expect(Encodings.charset_from_content_type(nil)).to be_nil+ expect(Encodings.charset_from_content_type("")).to be_nil+ end+ end++ describe ".charset_from_document" do+ it "reads the charset from a meta tag" do+ html = "<html><head><meta charset=\"UTF-8\"><title>x</title></head><body></body></html>"+ expect(Encodings.charset_from_document(html)).to eq("utf-8")+ end++ it "reads the charset from the content attribute" do+ html =+ '<html><head><meta http-equiv="Content-Type" content="text/html; charset=windows-1252"></head></html>'+ expect(Encodings.charset_from_document(html)).to eq("windows-1252")+ end++ it "ignores meta tags beyond the sniffing window" do+ html = +"<html><head>#{"x" * 1024}<meta charset='utf-8'></head></html>"+ expect(Encodings.charset_from_document(html)).to be_nil+ end++ it "returns nil when the document declares no charset" do+ expect(+ Encodings.charset_from_document("<html><head><title>x</title></head></html>"),+ ).to be_nil+ end+ end++ describe ".valid_utf8?" do+ it "returns true for UTF-8 bytes" do+ expect(Encodings.valid_utf8?("Iran’s".b)).to eq(true)+ end++ it "returns false for single-byte bytes" do+ expect(Encodings.valid_utf8?("caf\xE9".b)).to eq(false)+ end++ it "does not mutate the input string's encoding" do+ string = "Iran’s".dup.force_encoding(Encoding::Windows_1252)+ Encodings.valid_utf8?(string)+ expect(string.encoding).to eq(Encoding::Windows_1252)+ end+ end++ describe ".unicode_encoding?" do+ it "returns true for Unicode charsets" do+ expect(Encodings.unicode_encoding?("utf-8")).to eq(true)+ end++ it "returns false for single-byte charsets" do+ expect(Encodings.unicode_encoding?("windows-1252")).to eq(false)+ expect(Encodings.unicode_encoding?("iso-8859-1")).to eq(false)+ end+ endenddiff --git a/spec/models/topic_embed_spec.rb b/spec/models/topic_embed_spec.rbindex c106b3d087f..ea4580a505b 100644--- a/spec/models/topic_embed_spec.rb+++ b/spec/models/topic_embed_spec.rb@@ -95,6 +95,146 @@ RSpec.describe TopicEmbed doexpect(parsed.body.squish).to eq(expected.squish)end+ context "with a charset in the Content-Type header" do+ let(:html) { <<~HTML.b }+ <html>+ <head>+ <title>Iran’s seven-day offer has ‘significant poison pills’</title>+ </head>+ <body>+ <p>The deal has what diplomats call “significant poison pills” built into the text of the offer, and negotiators on both sides say the seven-day window may be shorter than the headline suggests.</p>+ </body>+ </html>+ HTML++ it "decodes the title using the declared charset" do+ parsed =+ TopicEmbed.parse_html(+ html,+ "https://blog.discourse.com/somepost.html",+ content_type: "text/html; charset=UTF-8",+ )+ expect(parsed.title).to eq("Iran’s seven-day offer has ‘significant poison pills’")+ end++ it "decodes the body using the declared charset" do+ parsed =+ TopicEmbed.parse_html(+ html,+ "https://blog.discourse.com/somepost.html",+ content_type: "text/html; charset=UTF-8",+ )+ expect(parsed.body).to include("“significant poison pills”")+ end+ end++ context "with a misleading single-byte charset in the header" do+ it "prefers the page's UTF-8 declaration when the bytes are valid UTF-8" do+ html = <<~HTML.b+ <html>+ <head>+ <meta charset="utf-8">+ <title>Iran’s seven-day offer has ‘significant poison pills’</title>+ </head>+ <body>+ <p>The deal has what diplomats call “significant poison pills” built into the text of the offer, and negotiators on both sides say the seven-day window may be shorter than the headline suggests.</p>+ </body>+ </html>+ HTML++ parsed =+ TopicEmbed.parse_html(+ html,+ "https://blog.discourse.com/somepost.html",+ content_type: "text/html; charset=ISO-8859-1",+ )++ expect(parsed.title).to eq("Iran’s seven-day offer has ‘significant poison pills’")+ expect(parsed.body).to include("“significant poison pills”")+ end++ it "keeps decoding a genuine single-byte page as declared" do+ html =+ +"<html><head><title>caf\xE9</title></head><body><p>Diplomats on both sides say the offer may be withdrawn before the weekend.</p></body></html>".b++ parsed =+ TopicEmbed.parse_html(+ html,+ "https://blog.discourse.com/somepost.html",+ content_type: "text/html; charset=ISO-8859-1",+ )++ expect(parsed.title).to eq("café")+ end++ it "keeps the header charset when the page declares none" do+ html = <<~HTML.b+ <html>+ <head>+ <title>Iran’s seven-day offer has ‘significant poison pills’</title>+ </head>+ <body><p>Diplomats on both sides say the offer may be withdrawn before the weekend.</p></body>+ </html>+ HTML++ parsed =+ TopicEmbed.parse_html(+ html,+ "https://blog.discourse.com/somepost.html",+ content_type: "text/html; charset=ISO-8859-1",+ )++ expect(parsed.title).to eq(+ "Iranâ\u0080\u0099s seven-day offer has â\u0080\u0098significant poison pillsâ\u0080\u0099",+ )+ end+ end++ it "falls back to a meta charset when no Content-Type header is present" do+ html = <<~HTML.b+ <html>+ <head>+ <meta charset="utf-8">+ <title>Iran’s seven-day offer</title>+ </head>+ <body><p>Diplomats on both sides say the offer may be withdrawn before the weekend.</p></body>+ </html>+ HTML++ parsed = TopicEmbed.parse_html(html, "https://blog.discourse.com/somepost.html")++ expect(parsed.title).to eq("Iran’s seven-day offer")+ end++ it "degrades to sniffing when the header charset has no UTF-8 converter" do+ html =+ +"<html><head><title>News headline</title></head><body><p>Diplomats on both sides say the offer may be withdrawn before the weekend.</p></body></html>".b++ parsed =+ TopicEmbed.parse_html(+ html,+ "https://blog.discourse.com/somepost.html",+ content_type: "text/html; charset=UTF-7",+ )++ expect(parsed.title).to eq("News headline")+ end++ it "sniffs the whole document when the declared charset mis-parses the bytes" do+ html =+ +"<html><head><title>News headline</title></head><body><p>The deal has poison pills built into the text of the offer and negotiators say the seven-day window may be shorter.</p></body></html>".b++ parsed =+ TopicEmbed.parse_html(+ html,+ "https://blog.discourse.com/somepost.html",+ content_type: "text/html; charset=UTF-32BE",+ )++ expect(parsed.title).to eq("News headline")+ expect(parsed.body).to include("shorter")+ end+context "when creating a post" dolet!(:post) { TopicEmbed.import(user, url, title, contents) }let(:topic_embed) { TopicEmbed.find_by(post: post) }@@ -657,6 +797,28 @@ RSpec.describe TopicEmbed doendend+ context "with a charset declared in the HTTP Content-Type header" do+ let(:url) { "http://eviltrout.com/123" }+ let(:contents) do+ "<title>Iran’s seven-day offer has ‘significant poison pills’</title><body>some content here</body>"+ end++ before do+ stub_request(:get, url).to_return(+ status: 200,+ headers: {+ "Content-Type" => "text/html; charset=UTF-8",+ },+ body: contents.b,+ )+ end++ it "decodes the title as UTF-8" do+ response = TopicEmbed.find_remote(url)+ expect(response.title).to eq("Iran’s seven-day offer has ‘significant poison pills’")+ end+ end+context 'with post with allowed classes "foo" and "emoji"' dofab!(:user)let(:url) { "http://eviltrout.com/123" }