A platform for community discussion. Free, open, simple.
DEV: skip beacon page views from known crawler ASNs (#39437)
Krzysztof Kotlarek6f1e0a8e3881
diff --git a/lib/crawler_detection.rb b/lib/crawler_detection.rbindex 6efd1c872d2..471aea99535 100644--- a/lib/crawler_detection.rb+++ b/lib/crawler_detection.rb@@ -3,6 +3,23 @@module CrawlerDetectionWAYBACK_MACHINE_URL = "archive.org"+ # Source: https://ipinfo.io/tags/crawler+ CRAWLER_ASNS =+ Set.new(+ [+ 55_967, # Baidu+ 140_577, # Ahrefs+ 210_743, # Babbar+ 7_941, # Internet Archive+ 13_238, # Yandex+ 32_934, # Facebook/Meta+ 15_169, # Google+ 396_982, # Google Cloud+ 8_075, # Microsoft+ 136_907, # Huawei Cloud+ ],+ ).freeze+def self.to_matcher(string, type: nil)escaped = string.split("|").map { |agent| Regexp.escape(agent) }.join("|")@@ -51,6 +68,11 @@ module CrawlerDetectionendend+ def self.crawler_ip?(ip)+ return false if ip.blank?+ CRAWLER_ASNS.include?(DiscourseIpInfo.get(ip)[:asn])+ end+def self.show_browser_update?(user_agent)return false if SiteSetting.browser_update_user_agents.blank?diff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rbindex 632355980b2..2a57f348963 100644--- a/lib/middleware/request_tracker.rb+++ b/lib/middleware/request_tracker.rb@@ -147,7 +147,7 @@ class Middleware::RequestTrackerdata[:current_user_id],)end- elsif !SiteSetting.login_required+ elsif !SiteSetting.login_required && !CrawlerDetection.crawler_ip?(data[:request_remote_ip])ApplicationRequest.increment!(:page_view_anon_browser_beacon)ApplicationRequest.increment!(:page_view_anon_browser_mobile_beacon) if data[:is_mobile]diff --git a/spec/lib/crawler_detection_spec.rb b/spec/lib/crawler_detection_spec.rbindex 16c88a9176a..c6926635da9 100644--- a/spec/lib/crawler_detection_spec.rb+++ b/spec/lib/crawler_detection_spec.rb@@ -158,6 +158,31 @@ RSpec.describe CrawlerDetection doendend+ describe ".crawler_ip?" do+ it "returns false when the IP is blank" do+ expect(CrawlerDetection.crawler_ip?(nil)).to eq(false)+ expect(CrawlerDetection.crawler_ip?("")).to eq(false)+ end++ it "returns false when the resolved ASN is not in CRAWLER_ASNS" do+ DiscourseIpInfo.stubs(:get).with("1.2.3.4").returns({ asn: 1 })+ expect(CrawlerDetection.crawler_ip?("1.2.3.4")).to eq(false)+ end++ it "returns false when the resolved ASN is missing" do+ DiscourseIpInfo.stubs(:get).with("1.2.3.4").returns({})+ expect(CrawlerDetection.crawler_ip?("1.2.3.4")).to eq(false)+ end++ it "returns true when the resolved ASN is in CRAWLER_ASNS" do+ DiscourseIpInfo+ .stubs(:get)+ .with("1.2.3.4")+ .returns({ asn: CrawlerDetection::CRAWLER_ASNS.first })+ expect(CrawlerDetection.crawler_ip?("1.2.3.4")).to eq(true)+ end+ end+describe ".is_blocked_crawler?" doit "is false if user agent is a crawler and no allowlist or blocklist is defined" doexpect(CrawlerDetection.is_blocked_crawler?("Twitterbot")).to eq(false)diff --git a/spec/lib/middleware/request_tracker_spec.rb b/spec/lib/middleware/request_tracker_spec.rbindex 8cbf660f9b2..db68a222a26 100644--- a/spec/lib/middleware/request_tracker_spec.rb+++ b/spec/lib/middleware/request_tracker_spec.rb@@ -862,6 +862,24 @@ RSpec.describe Middleware::RequestTracker doexpect(ApplicationRequest.page_view_anon_browser_beacon.first).to be_nilend+ it "skips beacon page view when the remote IP resolves to a crawler ASN" do+ DiscourseIpInfo.stubs(:get).returns({ asn: CrawlerDetection::CRAWLER_ASNS.first })+ middleware = Middleware::RequestTracker.new(lambda { |env| [200, {}, ["OK"]] })+ middleware.call(beacon_env({}, { "action_dispatch.remote_ip" => "1.2.3.4" }))+ CachedCounting.flush++ expect(ApplicationRequest.page_view_anon_browser_beacon.first).to be_nil+ end++ it "counts beacon page view when the remote IP is not a crawler ASN" do+ DiscourseIpInfo.stubs(:get).returns({ asn: 1 })+ middleware = Middleware::RequestTracker.new(lambda { |env| [200, {}, ["OK"]] })+ middleware.call(beacon_env({}, { "action_dispatch.remote_ip" => "1.2.3.4" }))+ CachedCounting.flush++ expect(ApplicationRequest.page_view_anon_browser_beacon.first.count).to eq(1)+ end+context "when SiteSetting.use_beacon_for_browser_page_views is false" dobefore { SiteSetting.use_beacon_for_browser_page_views = false }