A platform for community discussion. Free, open, simple.
DEV: move crawler ASN list to hidden site setting (#39480)
Krzysztof Kotlarekc7cb7388c5cd
diff --git a/config/site_settings.yml b/config/site_settings.ymlindex 7f993c3e897..82d88c599ee 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -2793,6 +2793,11 @@ security:crawler_check_bypass_agents:hidden: truedefault: "cubot|MetaIAB"+ crawler_asns:+ hidden: true+ default: "55967"+ type: list+ list_type: compactcors_origins:default: ""type: listdiff --git a/lib/crawler_detection.rb b/lib/crawler_detection.rbindex 471aea99535..632eec33148 100644--- a/lib/crawler_detection.rb+++ b/lib/crawler_detection.rb@@ -3,23 +3,6 @@module CrawlerDetectionWAYBACK_MACHINE_URL = "archive.org"- # Source: https://ipinfo.io/tags/crawler- CRAWLER_ASNS =- Set.new(- [- 55_967, # Baidu- 140_577, # Ahrefs- 210_743, # Babbar- 7_941, # Internet Archive- 13_238, # Yandex- 32_934, # Facebook/Meta- 15_169, # Google- 396_982, # Google Cloud- 8_075, # Microsoft- 136_907, # Huawei Cloud- ],- ).freeze-def self.to_matcher(string, type: nil)escaped = string.split("|").map { |agent| Regexp.escape(agent) }.join("|")@@ -70,7 +53,9 @@ module CrawlerDetectiondef self.crawler_ip?(ip)return false if ip.blank?- CRAWLER_ASNS.include?(DiscourseIpInfo.get(ip)[:asn])+ asn = DiscourseIpInfo.get(ip)[:asn]+ return false if asn.blank?+ SiteSetting.crawler_asns_map.include?(asn.to_s)enddef self.show_browser_update?(user_agent)diff --git a/spec/lib/crawler_detection_spec.rb b/spec/lib/crawler_detection_spec.rbindex c6926635da9..a85feabe65e 100644--- a/spec/lib/crawler_detection_spec.rb+++ b/spec/lib/crawler_detection_spec.rb@@ -164,7 +164,7 @@ RSpec.describe CrawlerDetection doexpect(CrawlerDetection.crawler_ip?("")).to eq(false)end- it "returns false when the resolved ASN is not in CRAWLER_ASNS" do+ it "returns false when the resolved ASN is not in crawler_asns site setting" doDiscourseIpInfo.stubs(:get).with("1.2.3.4").returns({ asn: 1 })expect(CrawlerDetection.crawler_ip?("1.2.3.4")).to eq(false)end@@ -174,11 +174,11 @@ RSpec.describe CrawlerDetection doexpect(CrawlerDetection.crawler_ip?("1.2.3.4")).to eq(false)end- it "returns true when the resolved ASN is in CRAWLER_ASNS" do+ it "returns true when the resolved ASN is in crawler_asns site setting" doDiscourseIpInfo.stubs(:get).with("1.2.3.4")- .returns({ asn: CrawlerDetection::CRAWLER_ASNS.first })+ .returns({ asn: SiteSetting.crawler_asns_map.first.to_i })expect(CrawlerDetection.crawler_ip?("1.2.3.4")).to eq(true)endenddiff --git a/spec/lib/middleware/request_tracker_spec.rb b/spec/lib/middleware/request_tracker_spec.rbindex db68a222a26..3c6f34e61e4 100644--- a/spec/lib/middleware/request_tracker_spec.rb+++ b/spec/lib/middleware/request_tracker_spec.rb@@ -863,7 +863,7 @@ RSpec.describe Middleware::RequestTracker doendit "skips beacon page view when the remote IP resolves to a crawler ASN" do- DiscourseIpInfo.stubs(:get).returns({ asn: CrawlerDetection::CRAWLER_ASNS.first })+ DiscourseIpInfo.stubs(:get).returns({ asn: SiteSetting.crawler_asns_map.first.to_i })middleware = Middleware::RequestTracker.new(lambda { |env| [200, {}, ["OK"]] })middleware.call(beacon_env({}, { "action_dispatch.remote_ip" => "1.2.3.4" }))CachedCounting.flush