Skip to content

A platform for community discussion. Free, open, simple.

DEV: Use the Content-Type charset when encoding is missing from the document metadata when parsing companion topic pages (#44695)

Juan David Martínez Cubillos306221a727ad
diff --git a/app/models/topic_embed.rb b/app/models/topic_embed.rb
index a3395e5dc27..bbd744da3a5 100644
--- a/app/models/topic_embed.rb
+++ b/app/models/topic_embed.rb
@@ -192,17 +192,27 @@ class TopicEmbed < ActiveRecord::Base
return if uri.blank?
begin
- html = FinalDestination::HTTP.get(uri)
+ response = FinalDestination::HTTP.get_response(uri)
rescue OpenURI::HTTPError, Net::OpenTimeout, FinalDestination::SSRFDetector::DisallowedIpError
return
end
- parse_html(html, uri.to_s)
+ parse_html(response.body, uri.to_s, content_type: response["content-type"])
end
- def self.parse_html(html, url)
+ def self.parse_html(html, url, content_type: nil)
require "ruby-readability"
+ encoding = Encodings.charset_from_content_type(content_type)
+
+ # A transport-level charset can misrepresent the document. When the response
+ # header declares a non-Unicode charset but the page itself declares UTF-8
+ # and the bytes are valid UTF-8, the page's declaration wins.
+ if encoding.present? && !Encodings.unicode_encoding?(encoding) &&
+ Encodings.charset_from_document(html) == "utf-8" && Encodings.valid_utf8?(html)
+ encoding = "utf-8"
+ end
+
opts = {
tags: %w[
div
@@ -237,12 +247,19 @@ class TopicEmbed < ActiveRecord::Base
opts[
:blacklist
] = SiteSetting.blocked_embed_selectors if SiteSetting.blocked_embed_selectors.present?
+ opts[:encoding] = encoding if encoding.present?
allowed_embed_classnames =
SiteSetting.allowed_embed_classnames if SiteSetting.allowed_embed_classnames.present?
response = FetchResponse.new
- raw_doc = Nokogiri.HTML5(html)
+ raw_doc =
+ begin
+ Nokogiri.HTML5(html, encoding: encoding)
+ rescue ArgumentError, EncodingError
+ opts.delete(:encoding)
+ Nokogiri.HTML5(html)
+ end
response.url = url
diff --git a/lib/encodings.rb b/lib/encodings.rb
index b7e42fd2584..83e797999af 100644
--- a/lib/encodings.rb
+++ b/lib/encodings.rb
@@ -12,4 +12,48 @@ module Encodings
string.presence&.delete_prefix!(BOM)
string
end
+
+ # Returns the lowercase charset name declared in an HTTP Content-Type header,
+ # or nil when absent or unsupported. HTML parsers like Nokogiri only receive
+ # the raw body and cannot see transport-level headers, so callers must pass
+ # the extracted charset along explicitly.
+ def self.charset_from_content_type(content_type)
+ return if content_type.blank?
+
+ charset = content_type[/\bcharset\s*=\s*"?([a-z0-9_-]+)"?/i, 1]
+ return if charset.blank?
+
+ encoding = Encoding.find(charset)
+ return if encoding.dummy?
+
+ # Ruby has no identity converter, so skip the probe for UTF-8 itself.
+ Encoding::Converter.new(encoding, Encoding::UTF_8) if encoding != Encoding::UTF_8
+
+ charset.downcase
+ rescue ArgumentError, Encoding::ConverterNotFoundError
+ nil
+ end
+
+ def self.charset_from_document(html)
+ head = html.to_s.byteslice(0, 1024).to_s
+ charset = head[/<meta[^>]*\bcharset\s*=\s*["']?([a-z0-9_-]+)/i, 1]
+ charset&.downcase
+ end
+
+ def self.valid_utf8?(string)
+ string.to_s.dup.force_encoding(Encoding::UTF_8).valid_encoding?
+ end
+
+ def self.unicode_encoding?(charset)
+ encoding = Encoding.find(charset)
+ [
+ Encoding::UTF_8,
+ Encoding::UTF_16LE,
+ Encoding::UTF_16BE,
+ Encoding::UTF_32LE,
+ Encoding::UTF_32BE,
+ ].include?(encoding)
+ rescue ArgumentError
+ false
+ end
end
diff --git a/lib/retrieve_title.rb b/lib/retrieve_title.rb
index 8ec16002d27..4309b198983 100644
--- a/lib/retrieve_title.rb
+++ b/lib/retrieve_title.rb
@@ -27,8 +27,9 @@ module RetrieveTitle
doc = nil
begin
doc = Nokogiri.HTML5(html, encoding:)
- rescue ArgumentError
- # invalid HTML (Eg: too many attributes, status tree too deep) - ignore
+ rescue ArgumentError, EncodingError
+ # invalid HTML (Eg: too many attributes, status tree too deep) or an
+ # unsupported encoding - ignore
# Error in nokogumbo is not specialized, uses generic ArgumentError
# see: https://www.rubydoc.info/gems/nokogiri/Nokogiri/HTML5#label-Error+reporting
end
@@ -102,12 +103,7 @@ module RetrieveTitle
current = chunk
end
- if !encoding && content_type = _response["content-type"]&.strip&.downcase
- if content_type =~ /charset="?([a-z0-9_-]+)"?/
- encoding = Regexp.last_match(1)
- encoding = nil if !Encoding.list.map(&:name).map(&:downcase).include?(encoding)
- end
- end
+ encoding ||= Encodings.charset_from_content_type(_response["content-type"])
max_size = max_chunk_size(uri) * 1024
title = extract_title(current, encoding)
diff --git a/spec/lib/encodings_spec.rb b/spec/lib/encodings_spec.rb
index 3db2e416f3f..ddaca4500c5 100644
--- a/spec/lib/encodings_spec.rb
+++ b/spec/lib/encodings_spec.rb
@@ -8,4 +8,93 @@ RSpec.describe Encodings do
it "replaces invalid bytes" do
expect(Encodings.force_utf8("caf\xFFe")).to eq("cafe")
end
+
+ describe ".charset_from_content_type" do
+ it "returns the lowercase charset from the header" do
+ expect(Encodings.charset_from_content_type("text/html; charset=UTF-8")).to eq("utf-8")
+ end
+
+ it "returns the charset when quoted" do
+ expect(Encodings.charset_from_content_type('text/html; charset="utf-8"')).to eq("utf-8")
+ end
+
+ it "returns the charset when no space precedes it" do
+ expect(Encodings.charset_from_content_type("text/html;charset=euc-jp")).to eq("euc-jp")
+ end
+
+ it "returns the charset when whitespace surrounds the equals sign" do
+ expect(Encodings.charset_from_content_type("text/html; charset = Shift_JIS")).to eq(
+ "shift_jis",
+ )
+ end
+
+ it "returns nil when no charset is declared" do
+ expect(Encodings.charset_from_content_type("text/html")).to be_nil
+ end
+
+ it "returns nil when the charset is unsupported" do
+ expect(Encodings.charset_from_content_type("text/html; charset=not-an-encoding")).to be_nil
+ end
+
+ it "returns nil for dummy encodings without a UTF-8 converter" do
+ expect(Encodings.charset_from_content_type("text/html; charset=UTF-7")).to be_nil
+ expect(Encodings.charset_from_content_type("text/html; charset=UTF-16")).to be_nil
+ end
+
+ it "returns nil when the header is blank" do
+ expect(Encodings.charset_from_content_type(nil)).to be_nil
+ expect(Encodings.charset_from_content_type("")).to be_nil
+ end
+ end
+
+ describe ".charset_from_document" do
+ it "reads the charset from a meta tag" do
+ html = "<html><head><meta charset=\"UTF-8\"><title>x</title></head><body></body></html>"
+ expect(Encodings.charset_from_document(html)).to eq("utf-8")
+ end
+
+ it "reads the charset from the content attribute" do
+ html =
+ '<html><head><meta http-equiv="Content-Type" content="text/html; charset=windows-1252"></head></html>'
+ expect(Encodings.charset_from_document(html)).to eq("windows-1252")
+ end
+
+ it "ignores meta tags beyond the sniffing window" do
+ html = +"<html><head>#{"x" * 1024}<meta charset='utf-8'></head></html>"
+ expect(Encodings.charset_from_document(html)).to be_nil
+ end
+
+ it "returns nil when the document declares no charset" do
+ expect(
+ Encodings.charset_from_document("<html><head><title>x</title></head></html>"),
+ ).to be_nil
+ end
+ end
+
+ describe ".valid_utf8?" do
+ it "returns true for UTF-8 bytes" do
+ expect(Encodings.valid_utf8?("Iran’s".b)).to eq(true)
+ end
+
+ it "returns false for single-byte bytes" do
+ expect(Encodings.valid_utf8?("caf\xE9".b)).to eq(false)
+ end
+
+ it "does not mutate the input string's encoding" do
+ string = "Iran’s".dup.force_encoding(Encoding::Windows_1252)
+ Encodings.valid_utf8?(string)
+ expect(string.encoding).to eq(Encoding::Windows_1252)
+ end
+ end
+
+ describe ".unicode_encoding?" do
+ it "returns true for Unicode charsets" do
+ expect(Encodings.unicode_encoding?("utf-8")).to eq(true)
+ end
+
+ it "returns false for single-byte charsets" do
+ expect(Encodings.unicode_encoding?("windows-1252")).to eq(false)
+ expect(Encodings.unicode_encoding?("iso-8859-1")).to eq(false)
+ end
+ end
end
diff --git a/spec/models/topic_embed_spec.rb b/spec/models/topic_embed_spec.rb
index c106b3d087f..ea4580a505b 100644
--- a/spec/models/topic_embed_spec.rb
+++ b/spec/models/topic_embed_spec.rb
@@ -95,6 +95,146 @@ RSpec.describe TopicEmbed do
expect(parsed.body.squish).to eq(expected.squish)
end
+ context "with a charset in the Content-Type header" do
+ let(:html) { <<~HTML.b }
+ <html>
+ <head>
+ <title>Iran’s seven-day offer has ‘significant poison pills’</title>
+ </head>
+ <body>
+ <p>The deal has what diplomats call “significant poison pills” built into the text of the offer, and negotiators on both sides say the seven-day window may be shorter than the headline suggests.</p>
+ </body>
+ </html>
+ HTML
+
+ it "decodes the title using the declared charset" do
+ parsed =
+ TopicEmbed.parse_html(
+ html,
+ "https://blog.discourse.com/somepost.html",
+ content_type: "text/html; charset=UTF-8",
+ )
+ expect(parsed.title).to eq("Iran’s seven-day offer has ‘significant poison pills’")
+ end
+
+ it "decodes the body using the declared charset" do
+ parsed =
+ TopicEmbed.parse_html(
+ html,
+ "https://blog.discourse.com/somepost.html",
+ content_type: "text/html; charset=UTF-8",
+ )
+ expect(parsed.body).to include("“significant poison pills”")
+ end
+ end
+
+ context "with a misleading single-byte charset in the header" do
+ it "prefers the page's UTF-8 declaration when the bytes are valid UTF-8" do
+ html = <<~HTML.b
+ <html>
+ <head>
+ <meta charset="utf-8">
+ <title>Iran’s seven-day offer has ‘significant poison pills’</title>
+ </head>
+ <body>
+ <p>The deal has what diplomats call “significant poison pills” built into the text of the offer, and negotiators on both sides say the seven-day window may be shorter than the headline suggests.</p>
+ </body>
+ </html>
+ HTML
+
+ parsed =
+ TopicEmbed.parse_html(
+ html,
+ "https://blog.discourse.com/somepost.html",
+ content_type: "text/html; charset=ISO-8859-1",
+ )
+
+ expect(parsed.title).to eq("Iran’s seven-day offer has ‘significant poison pills’")
+ expect(parsed.body).to include("“significant poison pills”")
+ end
+
+ it "keeps decoding a genuine single-byte page as declared" do
+ html =
+ +"<html><head><title>caf\xE9</title></head><body><p>Diplomats on both sides say the offer may be withdrawn before the weekend.</p></body></html>".b
+
+ parsed =
+ TopicEmbed.parse_html(
+ html,
+ "https://blog.discourse.com/somepost.html",
+ content_type: "text/html; charset=ISO-8859-1",
+ )
+
+ expect(parsed.title).to eq("café")
+ end
+
+ it "keeps the header charset when the page declares none" do
+ html = <<~HTML.b
+ <html>
+ <head>
+ <title>Iran’s seven-day offer has ‘significant poison pills’</title>
+ </head>
+ <body><p>Diplomats on both sides say the offer may be withdrawn before the weekend.</p></body>
+ </html>
+ HTML
+
+ parsed =
+ TopicEmbed.parse_html(
+ html,
+ "https://blog.discourse.com/somepost.html",
+ content_type: "text/html; charset=ISO-8859-1",
+ )
+
+ expect(parsed.title).to eq(
+ "Iranâ\u0080\u0099s seven-day offer has â\u0080\u0098significant poison pillsâ\u0080\u0099",
+ )
+ end
+ end
+
+ it "falls back to a meta charset when no Content-Type header is present" do
+ html = <<~HTML.b
+ <html>
+ <head>
+ <meta charset="utf-8">
+ <title>Iran’s seven-day offer</title>
+ </head>
+ <body><p>Diplomats on both sides say the offer may be withdrawn before the weekend.</p></body>
+ </html>
+ HTML
+
+ parsed = TopicEmbed.parse_html(html, "https://blog.discourse.com/somepost.html")
+
+ expect(parsed.title).to eq("Iran’s seven-day offer")
+ end
+
+ it "degrades to sniffing when the header charset has no UTF-8 converter" do
+ html =
+ +"<html><head><title>News headline</title></head><body><p>Diplomats on both sides say the offer may be withdrawn before the weekend.</p></body></html>".b
+
+ parsed =
+ TopicEmbed.parse_html(
+ html,
+ "https://blog.discourse.com/somepost.html",
+ content_type: "text/html; charset=UTF-7",
+ )
+
+ expect(parsed.title).to eq("News headline")
+ end
+
+ it "sniffs the whole document when the declared charset mis-parses the bytes" do
+ html =
+ +"<html><head><title>News headline</title></head><body><p>The deal has poison pills built into the text of the offer and negotiators say the seven-day window may be shorter.</p></body></html>".b
+
+ parsed =
+ TopicEmbed.parse_html(
+ html,
+ "https://blog.discourse.com/somepost.html",
+ content_type: "text/html; charset=UTF-32BE",
+ )
+
+ expect(parsed.title).to eq("News headline")
+ expect(parsed.body).to include("shorter")
+ end
+
context "when creating a post" do
let!(:post) { TopicEmbed.import(user, url, title, contents) }
let(:topic_embed) { TopicEmbed.find_by(post: post) }
@@ -657,6 +797,28 @@ RSpec.describe TopicEmbed do
end
end
+ context "with a charset declared in the HTTP Content-Type header" do
+ let(:url) { "http://eviltrout.com/123" }
+ let(:contents) do
+ "<title>Iran’s seven-day offer has ‘significant poison pills’</title><body>some content here</body>"
+ end
+
+ before do
+ stub_request(:get, url).to_return(
+ status: 200,
+ headers: {
+ "Content-Type" => "text/html; charset=UTF-8",
+ },
+ body: contents.b,
+ )
+ end
+
+ it "decodes the title as UTF-8" do
+ response = TopicEmbed.find_remote(url)
+ expect(response.title).to eq("Iran’s seven-day offer has ‘significant poison pills’")
+ end
+ end
+
context 'with post with allowed classes "foo" and "emoji"' do
fab!(:user)
let(:url) { "http://eviltrout.com/123" }