Skip to content

A platform for community discussion. Free, open, simple.

DEV: skip beacon page views from known crawler ASNs (#39437)

Krzysztof Kotlarek6f1e0a8e3881
diff --git a/lib/crawler_detection.rb b/lib/crawler_detection.rb
index 6efd1c872d2..471aea99535 100644
--- a/lib/crawler_detection.rb
+++ b/lib/crawler_detection.rb
@@ -3,6 +3,23 @@
module CrawlerDetection
WAYBACK_MACHINE_URL = "archive.org"
+ # Source: https://ipinfo.io/tags/crawler
+ CRAWLER_ASNS =
+ Set.new(
+ [
+ 55_967, # Baidu
+ 140_577, # Ahrefs
+ 210_743, # Babbar
+ 7_941, # Internet Archive
+ 13_238, # Yandex
+ 32_934, # Facebook/Meta
+ 15_169, # Google
+ 396_982, # Google Cloud
+ 8_075, # Microsoft
+ 136_907, # Huawei Cloud
+ ],
+ ).freeze
+
def self.to_matcher(string, type: nil)
escaped = string.split("|").map { |agent| Regexp.escape(agent) }.join("|")
@@ -51,6 +68,11 @@ module CrawlerDetection
end
end
+ def self.crawler_ip?(ip)
+ return false if ip.blank?
+ CRAWLER_ASNS.include?(DiscourseIpInfo.get(ip)[:asn])
+ end
+
def self.show_browser_update?(user_agent)
return false if SiteSetting.browser_update_user_agents.blank?
diff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rb
index 632355980b2..2a57f348963 100644
--- a/lib/middleware/request_tracker.rb
+++ b/lib/middleware/request_tracker.rb
@@ -147,7 +147,7 @@ class Middleware::RequestTracker
data[:current_user_id],
)
end
- elsif !SiteSetting.login_required
+ elsif !SiteSetting.login_required && !CrawlerDetection.crawler_ip?(data[:request_remote_ip])
ApplicationRequest.increment!(:page_view_anon_browser_beacon)
ApplicationRequest.increment!(:page_view_anon_browser_mobile_beacon) if data[:is_mobile]
diff --git a/spec/lib/crawler_detection_spec.rb b/spec/lib/crawler_detection_spec.rb
index 16c88a9176a..c6926635da9 100644
--- a/spec/lib/crawler_detection_spec.rb
+++ b/spec/lib/crawler_detection_spec.rb
@@ -158,6 +158,31 @@ RSpec.describe CrawlerDetection do
end
end
+ describe ".crawler_ip?" do
+ it "returns false when the IP is blank" do
+ expect(CrawlerDetection.crawler_ip?(nil)).to eq(false)
+ expect(CrawlerDetection.crawler_ip?("")).to eq(false)
+ end
+
+ it "returns false when the resolved ASN is not in CRAWLER_ASNS" do
+ DiscourseIpInfo.stubs(:get).with("1.2.3.4").returns({ asn: 1 })
+ expect(CrawlerDetection.crawler_ip?("1.2.3.4")).to eq(false)
+ end
+
+ it "returns false when the resolved ASN is missing" do
+ DiscourseIpInfo.stubs(:get).with("1.2.3.4").returns({})
+ expect(CrawlerDetection.crawler_ip?("1.2.3.4")).to eq(false)
+ end
+
+ it "returns true when the resolved ASN is in CRAWLER_ASNS" do
+ DiscourseIpInfo
+ .stubs(:get)
+ .with("1.2.3.4")
+ .returns({ asn: CrawlerDetection::CRAWLER_ASNS.first })
+ expect(CrawlerDetection.crawler_ip?("1.2.3.4")).to eq(true)
+ end
+ end
+
describe ".is_blocked_crawler?" do
it "is false if user agent is a crawler and no allowlist or blocklist is defined" do
expect(CrawlerDetection.is_blocked_crawler?("Twitterbot")).to eq(false)
diff --git a/spec/lib/middleware/request_tracker_spec.rb b/spec/lib/middleware/request_tracker_spec.rb
index 8cbf660f9b2..db68a222a26 100644
--- a/spec/lib/middleware/request_tracker_spec.rb
+++ b/spec/lib/middleware/request_tracker_spec.rb
@@ -862,6 +862,24 @@ RSpec.describe Middleware::RequestTracker do
expect(ApplicationRequest.page_view_anon_browser_beacon.first).to be_nil
end
+ it "skips beacon page view when the remote IP resolves to a crawler ASN" do
+ DiscourseIpInfo.stubs(:get).returns({ asn: CrawlerDetection::CRAWLER_ASNS.first })
+ middleware = Middleware::RequestTracker.new(lambda { |env| [200, {}, ["OK"]] })
+ middleware.call(beacon_env({}, { "action_dispatch.remote_ip" => "1.2.3.4" }))
+ CachedCounting.flush
+
+ expect(ApplicationRequest.page_view_anon_browser_beacon.first).to be_nil
+ end
+
+ it "counts beacon page view when the remote IP is not a crawler ASN" do
+ DiscourseIpInfo.stubs(:get).returns({ asn: 1 })
+ middleware = Middleware::RequestTracker.new(lambda { |env| [200, {}, ["OK"]] })
+ middleware.call(beacon_env({}, { "action_dispatch.remote_ip" => "1.2.3.4" }))
+ CachedCounting.flush
+
+ expect(ApplicationRequest.page_view_anon_browser_beacon.first.count).to eq(1)
+ end
+
context "when SiteSetting.use_beacon_for_browser_page_views is false" do
before { SiteSetting.use_beacon_for_browser_page_views = false }