Skip to content

A platform for community discussion. Free, open, simple.

FIX: Skip non-HTML responses in `CrawlerHooks` middleware (#39610)

Rafael dos Santos Silvabc0843a80852
diff --git a/lib/middleware/crawler_hooks.rb b/lib/middleware/crawler_hooks.rb
index 00859a3ee60..bf07d9a7d29 100644
--- a/lib/middleware/crawler_hooks.rb
+++ b/lib/middleware/crawler_hooks.rb
@@ -11,6 +11,7 @@ module Middleware
status, headers, response = @app.call(env)
if status == 200 && headers["X-Discourse-Crawler-View"] &&
+ headers["Content-Type"]&.include?("text/html") &&
SiteSetting.content_localization_enabled &&
SiteSetting.content_localization_crawler_param
response = transform_response(request:, response:)
diff --git a/spec/lib/middleware/crawler_hooks_spec.rb b/spec/lib/middleware/crawler_hooks_spec.rb
index 4e180e9bb8f..1d4f0d67d7d 100644
--- a/spec/lib/middleware/crawler_hooks_spec.rb
+++ b/spec/lib/middleware/crawler_hooks_spec.rb
@@ -24,7 +24,29 @@ describe Middleware::CrawlerHooks do
let(:json_middleware) do
Middleware::CrawlerHooks.new(
lambda do |_|
- [200, { "Content-Type" => "application/json; charset=utf-8" }, ['{ "key": "value" }']]
+ [
+ 200,
+ {
+ "Content-Type" => "application/json; charset=utf-8",
+ "X-Discourse-Crawler-View" => "true",
+ },
+ ['{ "key": "value" }'],
+ ]
+ end,
+ )
+ end
+ let(:binary_middleware) do
+ Middleware::CrawlerHooks.new(
+ lambda do |_|
+ [
+ 200,
+ {
+ "Content-Type" => "application/zip",
+ "Content-Disposition" => "attachment; filename=\"file.zip\"",
+ "X-Discourse-Crawler-View" => "true",
+ },
+ ["PK\x03\x04binarydata".b],
+ ]
end,
)
end
@@ -71,13 +93,45 @@ describe Middleware::CrawlerHooks do
end
it "does not modify responses for non-HTML content types" do
- status, headers, response = json_middleware.call(env("HTTP_USER_AGENT" => crawler_user_agent))
+ SiteSetting.content_localization_enabled = true
+ SiteSetting.content_localization_crawler_param = true
+
+ status, headers, response =
+ json_middleware.call(
+ env(
+ :path => "https://discourse.site",
+ :params => {
+ Discourse::LOCALE_PARAM => "fr",
+ },
+ "HTTP_USER_AGENT" => crawler_user_agent,
+ ),
+ )
expect(status).to eq(200)
expect(headers["Content-Type"]).to include("application/json")
expect(response).to eq(['{ "key": "value" }'])
end
+ it "does not parse binary file downloads as HTML" do
+ SiteSetting.content_localization_enabled = true
+ SiteSetting.content_localization_crawler_param = true
+
+ status, headers, response =
+ binary_middleware.call(
+ env(
+ :path => "https://discourse.site/uploads/short-url/abc.zip",
+ :params => {
+ Discourse::LOCALE_PARAM => "fr",
+ },
+ "HTTP_USER_AGENT" => crawler_user_agent,
+ ),
+ )
+
+ expect(status).to eq(200)
+ expect(headers["Content-Type"]).to eq("application/zip")
+ expect(response).to eq(["PK\x03\x04binarydata".b])
+ end
+
it "does not modify responses for non-200 status codes" do
status, headers, response =
error_middleware.call(env("HTTP_USER_AGENT" => crawler_user_agent))