A platform for community discussion. Free, open, simple.
FIX: Skip non-HTML responses in `CrawlerHooks` middleware (#39610)
Rafael dos Santos Silvabc0843a80852
diff --git a/lib/middleware/crawler_hooks.rb b/lib/middleware/crawler_hooks.rbindex 00859a3ee60..bf07d9a7d29 100644--- a/lib/middleware/crawler_hooks.rb+++ b/lib/middleware/crawler_hooks.rb@@ -11,6 +11,7 @@ module Middlewarestatus, headers, response = @app.call(env)if status == 200 && headers["X-Discourse-Crawler-View"] &&+ headers["Content-Type"]&.include?("text/html") &&SiteSetting.content_localization_enabled &&SiteSetting.content_localization_crawler_paramresponse = transform_response(request:, response:)diff --git a/spec/lib/middleware/crawler_hooks_spec.rb b/spec/lib/middleware/crawler_hooks_spec.rbindex 4e180e9bb8f..1d4f0d67d7d 100644--- a/spec/lib/middleware/crawler_hooks_spec.rb+++ b/spec/lib/middleware/crawler_hooks_spec.rb@@ -24,7 +24,29 @@ describe Middleware::CrawlerHooks dolet(:json_middleware) doMiddleware::CrawlerHooks.new(lambda do |_|- [200, { "Content-Type" => "application/json; charset=utf-8" }, ['{ "key": "value" }']]+ [+ 200,+ {+ "Content-Type" => "application/json; charset=utf-8",+ "X-Discourse-Crawler-View" => "true",+ },+ ['{ "key": "value" }'],+ ]+ end,+ )+ end+ let(:binary_middleware) do+ Middleware::CrawlerHooks.new(+ lambda do |_|+ [+ 200,+ {+ "Content-Type" => "application/zip",+ "Content-Disposition" => "attachment; filename=\"file.zip\"",+ "X-Discourse-Crawler-View" => "true",+ },+ ["PK\x03\x04binarydata".b],+ ]end,)end@@ -71,13 +93,45 @@ describe Middleware::CrawlerHooks doendit "does not modify responses for non-HTML content types" do- status, headers, response = json_middleware.call(env("HTTP_USER_AGENT" => crawler_user_agent))+ SiteSetting.content_localization_enabled = true+ SiteSetting.content_localization_crawler_param = true++ status, headers, response =+ json_middleware.call(+ env(+ :path => "https://discourse.site",+ :params => {+ Discourse::LOCALE_PARAM => "fr",+ },+ "HTTP_USER_AGENT" => crawler_user_agent,+ ),+ )expect(status).to eq(200)expect(headers["Content-Type"]).to include("application/json")expect(response).to eq(['{ "key": "value" }'])end+ it "does not parse binary file downloads as HTML" do+ SiteSetting.content_localization_enabled = true+ SiteSetting.content_localization_crawler_param = true++ status, headers, response =+ binary_middleware.call(+ env(+ :path => "https://discourse.site/uploads/short-url/abc.zip",+ :params => {+ Discourse::LOCALE_PARAM => "fr",+ },+ "HTTP_USER_AGENT" => crawler_user_agent,+ ),+ )++ expect(status).to eq(200)+ expect(headers["Content-Type"]).to eq("application/zip")+ expect(response).to eq(["PK\x03\x04binarydata".b])+ end+it "does not modify responses for non-200 status codes" dostatus, headers, response =error_middleware.call(env("HTTP_USER_AGENT" => crawler_user_agent))