Skip to content

A platform for community discussion. Free, open, simple.

UX: Explain bot verification challenges in onebox error previews (#41764)

Régis Hanol622ba2f813c4
diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml
index bd3f11041c6..b38d2eb388c 100644
--- a/config/locales/server.en.yml
+++ b/config/locales/server.en.yml
@@ -384,6 +384,7 @@ en:
onebox:
invalid_address: "Sorry, we were unable to generate a preview for this web page, because the server '%{hostname}' could not be found. Instead of a preview, only a link will appear in your post. :cry:"
error_response: "Sorry, we were unable to generate a preview for this web page, because the web server returned an error code of %{status_code}. Instead of a preview, only a link will appear in your post. :cry:"
+ bot_challenge: "Sorry, we were unable to generate a preview for this web page, because the web server requires visitors to pass a verification step in a web browser. Instead of a preview, only a link will appear in your post. :cry:"
missing_data:
one: "Sorry, we were unable to generate a preview for this web page, because the following oEmbed / OpenGraph tag could not be found: %{missing_attributes}"
other: "Sorry, we were unable to generate a preview for this web page, because the following oEmbed / OpenGraph tags could not be found: %{missing_attributes}"
diff --git a/lib/final_destination.rb b/lib/final_destination.rb
index b7f76b1bdc1..20473305231 100644
--- a/lib/final_destination.rb
+++ b/lib/final_destination.rb
@@ -115,6 +115,10 @@ class FinalDestination
@limit < @max_redirects
end
+ def bot_challenge?
+ !!@bot_challenge
+ end
+
def request_headers
result = {
"User-Agent" => @user_agent,
@@ -384,6 +388,9 @@ class FinalDestination
# this is weird an exception seems better
@status = :failure
@status_code = response.status
+ @bot_challenge =
+ response.headers["x-amzn-waf-action"].present? ||
+ response.headers["cf-mitigated"] == "challenge"
log(:warn, "FinalDestination could not resolve URL (status #{response.status}): #{@uri}")
nil
diff --git a/lib/oneboxer.rb b/lib/oneboxer.rb
index 1cbef72076b..8e8c9125eb4 100644
--- a/lib/oneboxer.rb
+++ b/lib/oneboxer.rb
@@ -560,6 +560,8 @@ module Oneboxer
elsif (fd.status_code || uri.nil?) && available_strategies.present?
# Try a different oneboxing strategy, if we have any options left:
return compute_external_onebox(url, available_strategies)
+ elsif fd.bot_challenge?
+ args[:error_message] = I18n.t("errors.onebox.bot_challenge")
elsif fd.status_code
args[:error_message] = I18n.t("errors.onebox.error_response", status_code: fd.status_code)
end
diff --git a/spec/lib/final_destination_spec.rb b/spec/lib/final_destination_spec.rb
index 0fd792d8582..104aca4229e 100644
--- a/spec/lib/final_destination_spec.rb
+++ b/spec/lib/final_destination_spec.rb
@@ -539,6 +539,48 @@ RSpec.describe FinalDestination do
expect(final.status).to eq(:resolved)
end
end
+
+ context "when the response is a bot verification challenge" do
+ it "detects a WAF challenge header" do
+ fd_stub_request(:head, "https://eviltrout.com").to_return(
+ status: 202,
+ headers: {
+ "x-amzn-waf-action" => "challenge",
+ },
+ )
+
+ final = FinalDestination.new("https://eviltrout.com", opts)
+ expect(final.resolve).to be_nil
+ expect(final.status).to eq(:failure)
+ expect(final.status_code).to eq(202)
+ expect(final.bot_challenge?).to eq(true)
+ end
+
+ it "detects a CDN challenge header" do
+ fd_stub_request(:head, "https://eviltrout.com").to_return(
+ status: 403,
+ headers: {
+ "cf-mitigated" => "challenge",
+ },
+ )
+
+ final = FinalDestination.new("https://eviltrout.com", opts)
+ expect(final.resolve).to be_nil
+ expect(final.status).to eq(:failure)
+ expect(final.status_code).to eq(403)
+ expect(final.bot_challenge?).to eq(true)
+ end
+
+ it "does not report a challenge for a plain error response" do
+ fd_stub_request(:head, "https://eviltrout.com").to_return(status: 403)
+
+ final = FinalDestination.new("https://eviltrout.com", opts)
+ expect(final.resolve).to be_nil
+ expect(final.status).to eq(:failure)
+ expect(final.status_code).to eq(403)
+ expect(final.bot_challenge?).to eq(false)
+ end
+ end
end
describe "#get" do
diff --git a/spec/lib/oneboxer_spec.rb b/spec/lib/oneboxer_spec.rb
index 1234260031f..2b4f95daf24 100644
--- a/spec/lib/oneboxer_spec.rb
+++ b/spec/lib/oneboxer_spec.rb
@@ -16,6 +16,31 @@ RSpec.describe Oneboxer do
expect(Oneboxer.onebox("http://boom.com")).to eq("")
end
+ it "shows the bot challenge error message when the response is a verification challenge" do
+ url = "https://challenged.example.com/page"
+ %i[head get].each do |method|
+ stub_request(method, url).to_return(
+ status: 202,
+ headers: {
+ "x-amzn-waf-action" => "challenge",
+ },
+ )
+ end
+
+ expect(Oneboxer.preview(url, invalidate_oneboxes: true)).to include(
+ I18n.t("errors.onebox.bot_challenge").sub(" :cry:", ""),
+ )
+ end
+
+ it "shows the status code error message for a plain error response" do
+ url = "https://error.example.com/page"
+ %i[head get].each { |method| stub_request(method, url).to_return(status: 403) }
+
+ expect(Oneboxer.preview(url, invalidate_oneboxes: true)).to include(
+ I18n.t("errors.onebox.error_response", status_code: 403).sub(" :cry:", ""),
+ )
+ end
+
describe "#invalidate" do
let(:url) { "http://test.com" }
it "clears the cached preview for the onebox URL and the failed URL cache" do