A platform for community discussion. Free, open, simple.
UX: Explain bot verification challenges in onebox error previews (#41764)
Régis Hanol622ba2f813c4
diff --git a/config/locales/server.en.yml b/config/locales/server.en.ymlindex bd3f11041c6..b38d2eb388c 100644--- a/config/locales/server.en.yml+++ b/config/locales/server.en.yml@@ -384,6 +384,7 @@ en:onebox:invalid_address: "Sorry, we were unable to generate a preview for this web page, because the server '%{hostname}' could not be found. Instead of a preview, only a link will appear in your post. :cry:"error_response: "Sorry, we were unable to generate a preview for this web page, because the web server returned an error code of %{status_code}. Instead of a preview, only a link will appear in your post. :cry:"+ bot_challenge: "Sorry, we were unable to generate a preview for this web page, because the web server requires visitors to pass a verification step in a web browser. Instead of a preview, only a link will appear in your post. :cry:"missing_data:one: "Sorry, we were unable to generate a preview for this web page, because the following oEmbed / OpenGraph tag could not be found: %{missing_attributes}"other: "Sorry, we were unable to generate a preview for this web page, because the following oEmbed / OpenGraph tags could not be found: %{missing_attributes}"diff --git a/lib/final_destination.rb b/lib/final_destination.rbindex b7f76b1bdc1..20473305231 100644--- a/lib/final_destination.rb+++ b/lib/final_destination.rb@@ -115,6 +115,10 @@ class FinalDestination@limit < @max_redirectsend+ def bot_challenge?+ !!@bot_challenge+ end+def request_headersresult = {"User-Agent" => @user_agent,@@ -384,6 +388,9 @@ class FinalDestination# this is weird an exception seems better@status = :failure@status_code = response.status+ @bot_challenge =+ response.headers["x-amzn-waf-action"].present? ||+ response.headers["cf-mitigated"] == "challenge"log(:warn, "FinalDestination could not resolve URL (status #{response.status}): #{@uri}")nildiff --git a/lib/oneboxer.rb b/lib/oneboxer.rbindex 1cbef72076b..8e8c9125eb4 100644--- a/lib/oneboxer.rb+++ b/lib/oneboxer.rb@@ -560,6 +560,8 @@ module Oneboxerelsif (fd.status_code || uri.nil?) && available_strategies.present?# Try a different oneboxing strategy, if we have any options left:return compute_external_onebox(url, available_strategies)+ elsif fd.bot_challenge?+ args[:error_message] = I18n.t("errors.onebox.bot_challenge")elsif fd.status_codeargs[:error_message] = I18n.t("errors.onebox.error_response", status_code: fd.status_code)enddiff --git a/spec/lib/final_destination_spec.rb b/spec/lib/final_destination_spec.rbindex 0fd792d8582..104aca4229e 100644--- a/spec/lib/final_destination_spec.rb+++ b/spec/lib/final_destination_spec.rb@@ -539,6 +539,48 @@ RSpec.describe FinalDestination doexpect(final.status).to eq(:resolved)endend++ context "when the response is a bot verification challenge" do+ it "detects a WAF challenge header" do+ fd_stub_request(:head, "https://eviltrout.com").to_return(+ status: 202,+ headers: {+ "x-amzn-waf-action" => "challenge",+ },+ )++ final = FinalDestination.new("https://eviltrout.com", opts)+ expect(final.resolve).to be_nil+ expect(final.status).to eq(:failure)+ expect(final.status_code).to eq(202)+ expect(final.bot_challenge?).to eq(true)+ end++ it "detects a CDN challenge header" do+ fd_stub_request(:head, "https://eviltrout.com").to_return(+ status: 403,+ headers: {+ "cf-mitigated" => "challenge",+ },+ )++ final = FinalDestination.new("https://eviltrout.com", opts)+ expect(final.resolve).to be_nil+ expect(final.status).to eq(:failure)+ expect(final.status_code).to eq(403)+ expect(final.bot_challenge?).to eq(true)+ end++ it "does not report a challenge for a plain error response" do+ fd_stub_request(:head, "https://eviltrout.com").to_return(status: 403)++ final = FinalDestination.new("https://eviltrout.com", opts)+ expect(final.resolve).to be_nil+ expect(final.status).to eq(:failure)+ expect(final.status_code).to eq(403)+ expect(final.bot_challenge?).to eq(false)+ end+ endenddescribe "#get" dodiff --git a/spec/lib/oneboxer_spec.rb b/spec/lib/oneboxer_spec.rbindex 1234260031f..2b4f95daf24 100644--- a/spec/lib/oneboxer_spec.rb+++ b/spec/lib/oneboxer_spec.rb@@ -16,6 +16,31 @@ RSpec.describe Oneboxer doexpect(Oneboxer.onebox("http://boom.com")).to eq("")end+ it "shows the bot challenge error message when the response is a verification challenge" do+ url = "https://challenged.example.com/page"+ %i[head get].each do |method|+ stub_request(method, url).to_return(+ status: 202,+ headers: {+ "x-amzn-waf-action" => "challenge",+ },+ )+ end++ expect(Oneboxer.preview(url, invalidate_oneboxes: true)).to include(+ I18n.t("errors.onebox.bot_challenge").sub(" :cry:", ""),+ )+ end++ it "shows the status code error message for a plain error response" do+ url = "https://error.example.com/page"+ %i[head get].each { |method| stub_request(method, url).to_return(status: 403) }++ expect(Oneboxer.preview(url, invalidate_oneboxes: true)).to include(+ I18n.t("errors.onebox.error_response", status_code: 403).sub(" :cry:", ""),+ )+ end+describe "#invalidate" dolet(:url) { "http://test.com" }it "clears the cached preview for the onebox URL and the failed URL cache" do