Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Improve crawler detection of unengaged traffic (#42198)

Krzysztof Kotlarek749cb8b3c517
diff --git a/app/models/browser_pageview_event_score.rb b/app/models/browser_pageview_event_score.rb
index 8f0462677a5..34934d3eb6f 100644
--- a/app/models/browser_pageview_event_score.rb
+++ b/app/models/browser_pageview_event_score.rb
@@ -8,16 +8,19 @@ end
#
# Table name: browser_pageview_event_scores
#
-# id :bigint not null, primary key
-# automation_ua_score :integer default(0), not null
-# churn_score :integer default(0), not null
-# engagement_score :integer default(0), not null
-# ip_rotation_score :integer default(0), not null
-# known_asn_score :integer default(0), not null
-# rapid_nav_score :integer default(0), not null
-# referrer_score :integer default(0), not null
-# velocity_score :integer default(0), not null
-# event_id :bigint not null
+# id :bigint not null, primary key
+# automation_ua_score :integer default(0), not null
+# churn_score :integer default(0), not null
+# datacenter_asn_score :integer default(0), not null
+# engagement_score :integer default(0), not null
+# ip_rotation_score :integer default(0), not null
+# known_asn_score :integer default(0), not null
+# rapid_nav_score :integer default(0), not null
+# referrer_score :integer default(0), not null
+# single_request_no_referrer_score :integer default(0), not null
+# stale_browser_score :integer default(0), not null
+# velocity_score :integer default(0), not null
+# event_id :bigint not null
#
# Indexes
#
diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml
index 15db20584c3..7f67d9d316f 100644
--- a/config/locales/server.en.yml
+++ b/config/locales/server.en.yml
@@ -2075,6 +2075,9 @@ en:
allowed_iframes: "A list of iframe src URL prefixes that Discourse can safely allow in posts. * can be used to match any number of non-/ characters."
allowed_crawler_user_agents: "User agents of web crawlers that should be allowed to access the site. WARNING! SETTING THIS WILL DISALLOW ALL CRAWLERS NOT LISTED HERE!"
blocked_crawler_user_agents: "Unique case insensitive word in the user agent string identifying web crawlers that should not be allowed to access the site. Does not apply if allowlist is defined."
+ crawler_asns: "ASNs operated by verified web crawlers. Traffic from these networks is excluded from browser pageview reporting and receives a high crawler score."
+ crawler_detection_datacenter_asns: "ASNs that provide cloud, hosting, or data center services. These networks are not necessarily crawlers: their traffic receives a low-confidence crawler score and is still counted as browser traffic."
+ crawler_stale_chromium_major_version_cutoff: "Chromium major versions at or below this value in unengaged sessions receive a crawler score. Update it only after reviewing traffic; an outdated cutoff becomes less sensitive, not more aggressive."
slow_down_crawler_user_agents: "User agents of web crawlers that should be rate limited as configured in the {{setting:slow_down_crawler_rate}} setting. Each value must be at least 3 characters long."
slow_down_crawler_rate: "If {{setting:slow_down_crawler_user_agents}} is specified this rate will apply to all the crawlers (number of seconds delay between requests)"
llms_txt: "Upload a text file to be served at /llms.txt for LLM crawlers. See <a href='https://llmstxt.org/' target='_blank'>llmstxt.org</a> for more information."
diff --git a/config/site_settings.yml b/config/site_settings.yml
index 913d550c6d1..bb20202fc02 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -659,6 +659,12 @@ basic:
crawler_automation_user_agents:
default: "HeadlessChrome|Playwright|Puppeteer|Selenium|PhantomJS|jsdom"
hidden: true
+ crawler_stale_chromium_major_version_cutoff:
+ default: 138
+ type: integer
+ min: 1
+ max: 999
+ hidden: true
interface_color_selector:
client: true
enum: "InterfaceColorSelectorSetting"
@@ -3118,16 +3124,23 @@ security:
crawler_check_bypass_agents:
hidden: true
default: "cubot|MetaIAB"
- # 16509/14618=AWS, 396982=Google Cloud, 15169/36040=Google, 8075=Microsoft,
- # 714=Apple, 13238=Yandex, 32934=Meta, 13335=Cloudflare,
- # 140577=Ahrefs, 210743=Babbar, 7941=Internet Archive,
- # 16276=OVH, 24940=Hetzner, 136907=Huawei Cloud, 51747=Internet Vikings,
- # 4134/4837=China Telecom/Unicom, 5089=Virgin, 55836=Reliance Jio,
- # 5410=Bouygues, 701=Verizon, 6805=Telefonica, 45899=VNPT
- # 55967=Baidu
+ # 38365/55967=Baidu, 140577=Ahrefs, 210743=Babbar, 7941=Internet Archive,
+ # 13238=Yandex
crawler_asns:
hidden: true
- default: "16509|16276|714|13238|14618|136907|32934|15169|4134|4837|24940|396982|8075|51747|140577|45899|210743|13335|36040|5089|55836|5410|701|7941|6805|55967"
+ default: "38365|55967|140577|210743|7941|13238"
+ type: list
+ list_type: compact
+ # 16509/14618=AWS, 16276=OVH, 136907/55990=Huawei Cloud,
+ # 24940=Hetzner, 396982=Google Cloud, 8075=Microsoft, 51747=Internet Vikings,
+ # 13335=Cloudflare, 45090=Tencent Cloud, 23724=China Telecom IDC,
+ # 37963=Alibaba Cloud, 55960=Sinnet, 203020=HostRoyale,
+ # 63835=China Telecom Changsha IDC, 58540=China Telecom Jinan IDC,
+ # 58541=China Telecom Qingdao IDC, 58563=China Telecom Hubei IDC,
+ # 58461=China Telecom Hangzhou IDC
+ crawler_detection_datacenter_asns:
+ hidden: true
+ default: "16509|16276|14618|136907|24940|396982|8075|51747|13335|45090|55990|23724|37963|55960|203020|63835|58540|58541|58563|58461"
type: list
list_type: compact
cors_origins:
diff --git a/db/migrate/20260731055703_add_crawler_signals_to_browser_pageview_event_scores.rb b/db/migrate/20260731055703_add_crawler_signals_to_browser_pageview_event_scores.rb
new file mode 100644
index 00000000000..9779c5b41a6
--- /dev/null
+++ b/db/migrate/20260731055703_add_crawler_signals_to_browser_pageview_event_scores.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class AddCrawlerSignalsToBrowserPageviewEventScores < ActiveRecord::Migration[8.0]
+ def change
+ add_column :browser_pageview_event_scores,
+ :datacenter_asn_score,
+ :smallint,
+ null: false,
+ default: 0
+ add_column :browser_pageview_event_scores,
+ :single_request_no_referrer_score,
+ :smallint,
+ null: false,
+ default: 0
+ add_column :browser_pageview_event_scores,
+ :stale_browser_score,
+ :smallint,
+ null: false,
+ default: 0
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index f2925f6d9fa..e9c0d44918a 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1992,7 +1992,10 @@ CREATE TABLE public.browser_pageview_event_scores (
rapid_nav_score smallint DEFAULT 0 NOT NULL,
referrer_score smallint DEFAULT 0 NOT NULL,
engagement_score smallint DEFAULT 0 NOT NULL,
- ip_rotation_score smallint DEFAULT 0 NOT NULL
+ ip_rotation_score smallint DEFAULT 0 NOT NULL,
+ datacenter_asn_score smallint DEFAULT 0 NOT NULL,
+ single_request_no_referrer_score smallint DEFAULT 0 NOT NULL,
+ stale_browser_score smallint DEFAULT 0 NOT NULL
);
@@ -23072,6 +23075,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groups
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
+('20260731055703'),
('20260729153343'),
('20260728162521'),
('20260728162516'),
diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rb
index 0e3a3fcd97c..c49b8ce7f5c 100644
--- a/lib/crawler_scorer.rb
+++ b/lib/crawler_scorer.rb
@@ -5,7 +5,11 @@ class CrawlerScorer
AUTOMATION_UA_SCORE = 100
- KNOWN_ASN_SCORE = 15
+ KNOWN_ASN_SCORE = 30
+ DATACENTER_ASN_SCORE = 10
+ SINGLE_REQUEST_NO_REFERRER_SCORE = 10
+ SINGLE_REQUEST_LOCALE_PARAM_BONUS = 5
+ STALE_BROWSER_SCORE = 5
VELOCITY_LOW = 150
VELOCITY_MEDIUM = 300
@@ -38,6 +42,8 @@ class CrawlerScorer
def self.score!(window_start:, window_end:)
crawler_asns = SiteSetting.crawler_asns_map.map(&:to_i)
+ crawler_detection_datacenter_asns =
+ SiteSetting.crawler_detection_datacenter_asns_map.map(&:to_i)
ActiveRecord::Base.transaction do
DB.exec(
@@ -46,9 +52,16 @@ class CrawlerScorer
window_end: window_end,
ua_regex: SiteSetting.crawler_automation_user_agents,
crawler_asns: crawler_asns,
+ crawler_detection_datacenter_asns: crawler_detection_datacenter_asns,
hostname: Discourse.current_hostname,
automation_ua_score: AUTOMATION_UA_SCORE,
known_asn_score: KNOWN_ASN_SCORE,
+ datacenter_asn_score: DATACENTER_ASN_SCORE,
+ single_request_no_referrer_score: SINGLE_REQUEST_NO_REFERRER_SCORE,
+ single_request_locale_param_bonus: SINGLE_REQUEST_LOCALE_PARAM_BONUS,
+ stale_browser_score: STALE_BROWSER_SCORE,
+ stale_chromium_major_version_cutoff:
+ SiteSetting.crawler_stale_chromium_major_version_cutoff,
velocity_low: VELOCITY_LOW,
velocity_medium: VELOCITY_MEDIUM,
velocity_high: VELOCITY_HIGH,
@@ -78,7 +91,7 @@ class CrawlerScorer
SQL = <<~SQL
WITH events AS (
- SELECT id, session_id, ip_address, user_agent, referrer, asn, created_at, source
+ SELECT id, session_id, ip_address, user_agent, referrer, asn, url, created_at, source
FROM browser_pageview_events
WHERE created_at >= :window_start
AND created_at < :window_end
@@ -150,6 +163,30 @@ class CrawlerScorer
WHEN e.asn = ANY(ARRAY[:crawler_asns]::int[]) THEN :known_asn_score
ELSE 0
END AS known_asn_score,
+ CASE
+ WHEN e.asn = ANY(ARRAY[:crawler_detection_datacenter_asns]::int[]) THEN :datacenter_asn_score
+ ELSE 0
+ END AS datacenter_asn_score,
+ CASE
+ WHEN iu.pageviews = 1
+ AND e.referrer IS NULL
+ AND se.session_id IS NULL
+ AND e.url ~ '[?&]#{Discourse::LOCALE_PARAM}(=|&|$)'
+ THEN :single_request_no_referrer_score + :single_request_locale_param_bonus
+ WHEN iu.pageviews = 1
+ AND e.referrer IS NULL
+ AND se.session_id IS NULL THEN :single_request_no_referrer_score
+ ELSE 0
+ END AS single_request_no_referrer_score,
+ CASE
+ WHEN se.session_id IS NULL
+ AND e.user_agent ~ 'Chrome/[0-9]{1,3}'
+ AND e.user_agent !~* 'HeadlessChrome'
+ AND (substring(e.user_agent FROM 'Chrome/([0-9]{1,3})'))::int
+ <= :stale_chromium_major_version_cutoff
+ THEN :stale_browser_score
+ ELSE 0
+ END AS stale_browser_score,
CASE
WHEN iu.pageviews >= :velocity_high THEN :velocity_high_score
WHEN iu.pageviews >= :velocity_medium THEN :velocity_medium_score
@@ -205,6 +242,9 @@ class CrawlerScorer
id,
automation_ua_score,
known_asn_score,
+ datacenter_asn_score,
+ single_request_no_referrer_score,
+ stale_browser_score,
velocity_score,
churn_score,
rapid_nav_score,
@@ -213,11 +253,11 @@ class CrawlerScorer
engagement_score,
GREATEST(
0,
- automation_ua_score + known_asn_score + velocity_score + churn_score
+ automation_ua_score + known_asn_score + datacenter_asn_score + single_request_no_referrer_score + stale_browser_score + velocity_score + churn_score
+ rapid_nav_score + ip_rotation_score + referrer_score + engagement_score
) AS score
FROM breakdown
- WHERE automation_ua_score + known_asn_score + velocity_score + churn_score
+ WHERE automation_ua_score + known_asn_score + datacenter_asn_score + single_request_no_referrer_score + stale_browser_score + velocity_score + churn_score
+ rapid_nav_score + ip_rotation_score + referrer_score > 0
),
@@ -230,6 +270,9 @@ class CrawlerScorer
RETURNING e.id,
t.automation_ua_score,
t.known_asn_score,
+ t.datacenter_asn_score,
+ t.single_request_no_referrer_score,
+ t.stale_browser_score,
t.velocity_score,
t.churn_score,
t.rapid_nav_score,
@@ -242,6 +285,9 @@ class CrawlerScorer
event_id,
automation_ua_score,
known_asn_score,
+ datacenter_asn_score,
+ single_request_no_referrer_score,
+ stale_browser_score,
velocity_score,
churn_score,
rapid_nav_score,
@@ -253,6 +299,9 @@ class CrawlerScorer
id,
automation_ua_score,
known_asn_score,
+ datacenter_asn_score,
+ single_request_no_referrer_score,
+ stale_browser_score,
velocity_score,
churn_score,
rapid_nav_score,
@@ -263,6 +312,9 @@ class CrawlerScorer
ON CONFLICT (event_id) DO UPDATE
SET automation_ua_score = EXCLUDED.automation_ua_score,
known_asn_score = EXCLUDED.known_asn_score,
+ datacenter_asn_score = EXCLUDED.datacenter_asn_score,
+ single_request_no_referrer_score = EXCLUDED.single_request_no_referrer_score,
+ stale_browser_score = EXCLUDED.stale_browser_score,
velocity_score = EXCLUDED.velocity_score,
churn_score = EXCLUDED.churn_score,
rapid_nav_score = EXCLUDED.rapid_nav_score,
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb
index a65f0e12943..c62705849b4 100644
--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb
+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb
@@ -1541,6 +1541,9 @@ module DiscourseDataExplorer
COUNT(*) AS pageviews,
MAX(s.automation_ua_score) AS automation_ua,
MAX(s.known_asn_score) AS known_asn,
+ MAX(s.datacenter_asn_score) AS datacenter_asn,
+ MAX(s.single_request_no_referrer_score) AS single_request_no_referrer,
+ MAX(s.stale_browser_score) AS stale_browser,
MAX(s.velocity_score) AS velocity,
MAX(s.churn_score) AS churn,
MAX(s.rapid_nav_score) AS rapid_nav,
@@ -1551,6 +1554,14 @@ module DiscourseDataExplorer
CONCAT_WS(', ',
CASE WHEN MAX(s.automation_ua_score) > 0 THEN 'automation UA' END,
CASE WHEN MAX(s.known_asn_score) > 0 THEN 'known crawler ASN' END,
+ CASE WHEN MAX(s.datacenter_asn_score) > 0 THEN 'datacenter ASN' END,
+ CASE
+ WHEN MAX(s.single_request_no_referrer_score) = #{CrawlerScorer::SINGLE_REQUEST_NO_REFERRER_SCORE + CrawlerScorer::SINGLE_REQUEST_LOCALE_PARAM_BONUS}
+ THEN 'single direct locale request (+' || MAX(s.single_request_no_referrer_score) || ')'
+ WHEN MAX(s.single_request_no_referrer_score) > 0
+ THEN 'single direct request (+' || MAX(s.single_request_no_referrer_score) || ')'
+ END,
+ CASE WHEN MAX(s.stale_browser_score) > 0 THEN 'stale Chromium (+' || MAX(s.stale_browser_score) || ')' END,
CASE WHEN MAX(s.velocity_score) > 0 THEN 'high velocity (+' || MAX(s.velocity_score) || ')' END,
CASE WHEN MAX(s.churn_score) > 0 THEN 'session churn (+' || MAX(s.churn_score) || ')' END,
CASE WHEN MAX(s.rapid_nav_score) > 0 THEN 'rapid navigation' END,
diff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rb
index 7536030eaae..c5319871ca2 100644
--- a/spec/lib/crawler_scorer_spec.rb
+++ b/spec/lib/crawler_scorer_spec.rb
@@ -35,10 +35,13 @@ RSpec.describe CrawlerScorer do
score!
- expect(event.reload.score).to eq(155)
+ expect(event.reload.score).to eq(170)
breakdown = event.browser_pageview_event_score
expect(breakdown.automation_ua_score).to eq(100)
- expect(breakdown.known_asn_score).to eq(15)
+ expect(breakdown.known_asn_score).to eq(30)
+ expect(breakdown.datacenter_asn_score).to eq(0)
+ expect(breakdown.single_request_no_referrer_score).to eq(0)
+ expect(breakdown.stale_browser_score).to eq(0)
expect(breakdown.velocity_score).to eq(0)
expect(breakdown.churn_score).to eq(0)
expect(breakdown.rapid_nav_score).to eq(0)
@@ -49,16 +52,103 @@ RSpec.describe CrawlerScorer do
it "does not score an event whose only signal is missing engagement" do
event = make_event
+
score!
+
expect(event.reload.score).to be_nil
expect(event.browser_pageview_event_score).to be_nil
end
- it "scores known crawler ASNs at +15" do
+ it "scores known crawler ASNs at +30" do
SiteSetting.crawler_asns = "12345"
event = make_event(asn: 12_345)
score!
+ expect(event.reload.score).to eq(70)
+ end
+
+ it "scores datacenter ASNs at +10" do
+ SiteSetting.crawler_asns = ""
+ SiteSetting.crawler_detection_datacenter_asns = "12345"
+ event = make_event(asn: 12_345)
+
+ score!
+
+ expect(event.reload.score).to eq(50)
+ expect(event.browser_pageview_event_score.datacenter_asn_score).to eq(10)
+ end
+
+ it "scores unengaged single direct requests at +10" do
+ event = make_event(referrer: nil)
+
+ score!
+
+ expect(event.reload.score).to eq(50)
+ expect(event.browser_pageview_event_score.single_request_no_referrer_score).to eq(10)
+ end
+
+ it "scores unengaged single direct requests with any translation parameter at +15" do
+ event = make_event(url: "/t/topic/1?source=bot&tl=made-up-locale", referrer: nil)
+
+ score!
+
expect(event.reload.score).to eq(55)
+ expect(event.browser_pageview_event_score.single_request_no_referrer_score).to eq(15)
+ end
+
+ it "scores unengaged Chromium versions at or below the configured cutoff at +5" do
+ SiteSetting.crawler_stale_chromium_major_version_cutoff = 138
+ event = make_event(user_agent: "Mozilla/5.0 Chrome/138.0.0.0 Safari/537.36")
+
+ score!
+
+ expect(event.reload.score).to eq(45)
+ expect(event.browser_pageview_event_score.stale_browser_score).to eq(5)
+ end
+
+ it "does not score Chromium versions above the configured cutoff" do
+ SiteSetting.crawler_stale_chromium_major_version_cutoff = 138
+ event = make_event(user_agent: "Mozilla/5.0 Chrome/139.0.0.0 Safari/537.36")
+
+ score!
+
+ expect(event.reload.score).to be_nil
+ expect(event.browser_pageview_event_score).to be_nil
+ end
+
+ it "does not score oversized Chromium major versions as stale" do
+ event = make_event(user_agent: "Mozilla/5.0 Chrome/99999999999.0.0.0 Safari/537.36")
+
+ score!
+
+ expect(event.reload.score).to be_nil
+ expect(event.browser_pageview_event_score).to be_nil
+ end
+
+ it "does not score stale Chromium versions when the session has interaction" do
+ SiteSetting.crawler_asns = "12345"
+ event = make_event(asn: 12_345, user_agent: "Mozilla/5.0 Chrome/125.0.0.0 Safari/537.36")
+ Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, click_events: 1)
+
+ score!
+
+ expect(event.reload.score).to eq(30)
+ expect(event.browser_pageview_event_score.stale_browser_score).to eq(0)
+ end
+
+ it "scores stale Chromium versions reported by Edge and Opera" do
+ SiteSetting.crawler_stale_chromium_major_version_cutoff = 138
+ edge_event = make_event(user_agent: "Mozilla/5.0 Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0")
+ opera_event = make_event(user_agent: "Mozilla/5.0 Chrome/125.0.0.0 Safari/537.36 OPR/110.0.0.0")
+
+ score!
+
+ expect([edge_event.reload.score, opera_event.reload.score]).to eq([45, 45])
+ expect(
+ [
+ edge_event.browser_pageview_event_score.stale_browser_score,
+ opera_event.browser_pageview_event_score.stale_browser_score,
+ ],
+ ).to eq([5, 5])
end
it "scores pageview velocity at or above VELOCITY_LOW threshold at +15" do
@@ -138,7 +228,7 @@ RSpec.describe CrawlerScorer do
).to contain_exactly(70)
end
- it "does not score a session that only ever uses two addresses" do
+ it "does not add an IP rotation score to a session that only uses two addresses" do
base = 30.minutes.ago
%w[10.0.0.1 10.0.0.2].each_with_index do |ip, i|
make_event(ip_address: ip, session_id: "handover-session", created_at: base + (i * 5).seconds)
@@ -151,7 +241,7 @@ RSpec.describe CrawlerScorer do
)
end
- it "does not score ip changes slow enough to be a network handover" do
+ it "does not add an IP rotation score to a slow network handover" do
base = 40.minutes.ago
%w[10.0.0.1 10.0.0.2 10.0.0.3].each_with_index do |ip, i|
make_event(ip_address: ip, session_id: "slow-session", created_at: base + (i * 11).minutes)
@@ -221,9 +311,10 @@ RSpec.describe CrawlerScorer do
score!
- expect(event.reload.score).to eq(15)
+ expect(event.reload.score).to eq(30)
breakdown = event.browser_pageview_event_score
- expect(breakdown.known_asn_score).to eq(15)
+ expect(breakdown.known_asn_score).to eq(30)
+ expect(breakdown.datacenter_asn_score).to eq(0)
expect(breakdown.engagement_score).to eq(0)
end
@@ -246,12 +337,12 @@ RSpec.describe CrawlerScorer do
event = make_event(asn: 12_345)
score!
- expect(event.reload.score).to eq(55)
+ expect(event.reload.score).to eq(70)
Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, key_events: 4)
score!
- expect(event.reload.score).to eq(55)
+ expect(event.reload.score).to eq(70)
end
it "scores each source but partitions velocity so transports do not inflate each other" do