Skip to content

A platform for community discussion. Free, open, simple.

DEV: move crawler ASN list to hidden site setting (#39480)

Krzysztof Kotlarekc7cb7388c5cd
diff --git a/config/site_settings.yml b/config/site_settings.yml
index 7f993c3e897..82d88c599ee 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -2793,6 +2793,11 @@ security:
crawler_check_bypass_agents:
hidden: true
default: "cubot|MetaIAB"
+ crawler_asns:
+ hidden: true
+ default: "55967"
+ type: list
+ list_type: compact
cors_origins:
default: ""
type: list
diff --git a/lib/crawler_detection.rb b/lib/crawler_detection.rb
index 471aea99535..632eec33148 100644
--- a/lib/crawler_detection.rb
+++ b/lib/crawler_detection.rb
@@ -3,23 +3,6 @@
module CrawlerDetection
WAYBACK_MACHINE_URL = "archive.org"
- # Source: https://ipinfo.io/tags/crawler
- CRAWLER_ASNS =
- Set.new(
- [
- 55_967, # Baidu
- 140_577, # Ahrefs
- 210_743, # Babbar
- 7_941, # Internet Archive
- 13_238, # Yandex
- 32_934, # Facebook/Meta
- 15_169, # Google
- 396_982, # Google Cloud
- 8_075, # Microsoft
- 136_907, # Huawei Cloud
- ],
- ).freeze
-
def self.to_matcher(string, type: nil)
escaped = string.split("|").map { |agent| Regexp.escape(agent) }.join("|")
@@ -70,7 +53,9 @@ module CrawlerDetection
def self.crawler_ip?(ip)
return false if ip.blank?
- CRAWLER_ASNS.include?(DiscourseIpInfo.get(ip)[:asn])
+ asn = DiscourseIpInfo.get(ip)[:asn]
+ return false if asn.blank?
+ SiteSetting.crawler_asns_map.include?(asn.to_s)
end
def self.show_browser_update?(user_agent)
diff --git a/spec/lib/crawler_detection_spec.rb b/spec/lib/crawler_detection_spec.rb
index c6926635da9..a85feabe65e 100644
--- a/spec/lib/crawler_detection_spec.rb
+++ b/spec/lib/crawler_detection_spec.rb
@@ -164,7 +164,7 @@ RSpec.describe CrawlerDetection do
expect(CrawlerDetection.crawler_ip?("")).to eq(false)
end
- it "returns false when the resolved ASN is not in CRAWLER_ASNS" do
+ it "returns false when the resolved ASN is not in crawler_asns site setting" do
DiscourseIpInfo.stubs(:get).with("1.2.3.4").returns({ asn: 1 })
expect(CrawlerDetection.crawler_ip?("1.2.3.4")).to eq(false)
end
@@ -174,11 +174,11 @@ RSpec.describe CrawlerDetection do
expect(CrawlerDetection.crawler_ip?("1.2.3.4")).to eq(false)
end
- it "returns true when the resolved ASN is in CRAWLER_ASNS" do
+ it "returns true when the resolved ASN is in crawler_asns site setting" do
DiscourseIpInfo
.stubs(:get)
.with("1.2.3.4")
- .returns({ asn: CrawlerDetection::CRAWLER_ASNS.first })
+ .returns({ asn: SiteSetting.crawler_asns_map.first.to_i })
expect(CrawlerDetection.crawler_ip?("1.2.3.4")).to eq(true)
end
end
diff --git a/spec/lib/middleware/request_tracker_spec.rb b/spec/lib/middleware/request_tracker_spec.rb
index db68a222a26..3c6f34e61e4 100644
--- a/spec/lib/middleware/request_tracker_spec.rb
+++ b/spec/lib/middleware/request_tracker_spec.rb
@@ -863,7 +863,7 @@ RSpec.describe Middleware::RequestTracker do
end
it "skips beacon page view when the remote IP resolves to a crawler ASN" do
- DiscourseIpInfo.stubs(:get).returns({ asn: CrawlerDetection::CRAWLER_ASNS.first })
+ DiscourseIpInfo.stubs(:get).returns({ asn: SiteSetting.crawler_asns_map.first.to_i })
middleware = Middleware::RequestTracker.new(lambda { |env| [200, {}, ["OK"]] })
middleware.call(beacon_env({}, { "action_dispatch.remote_ip" => "1.2.3.4" }))
CachedCounting.flush