A platform for community discussion. Free, open, simple.
FEATURE: Improve crawler detection of unengaged traffic (#42198)
Krzysztof Kotlarek749cb8b3c517
diff --git a/app/models/browser_pageview_event_score.rb b/app/models/browser_pageview_event_score.rbindex 8f0462677a5..34934d3eb6f 100644--- a/app/models/browser_pageview_event_score.rb+++ b/app/models/browser_pageview_event_score.rb@@ -8,16 +8,19 @@ end## Table name: browser_pageview_event_scores#-# id :bigint not null, primary key-# automation_ua_score :integer default(0), not null-# churn_score :integer default(0), not null-# engagement_score :integer default(0), not null-# ip_rotation_score :integer default(0), not null-# known_asn_score :integer default(0), not null-# rapid_nav_score :integer default(0), not null-# referrer_score :integer default(0), not null-# velocity_score :integer default(0), not null-# event_id :bigint not null+# id :bigint not null, primary key+# automation_ua_score :integer default(0), not null+# churn_score :integer default(0), not null+# datacenter_asn_score :integer default(0), not null+# engagement_score :integer default(0), not null+# ip_rotation_score :integer default(0), not null+# known_asn_score :integer default(0), not null+# rapid_nav_score :integer default(0), not null+# referrer_score :integer default(0), not null+# single_request_no_referrer_score :integer default(0), not null+# stale_browser_score :integer default(0), not null+# velocity_score :integer default(0), not null+# event_id :bigint not null## Indexes#diff --git a/config/locales/server.en.yml b/config/locales/server.en.ymlindex 15db20584c3..7f67d9d316f 100644--- a/config/locales/server.en.yml+++ b/config/locales/server.en.yml@@ -2075,6 +2075,9 @@ en:allowed_iframes: "A list of iframe src URL prefixes that Discourse can safely allow in posts. * can be used to match any number of non-/ characters."allowed_crawler_user_agents: "User agents of web crawlers that should be allowed to access the site. WARNING! SETTING THIS WILL DISALLOW ALL CRAWLERS NOT LISTED HERE!"blocked_crawler_user_agents: "Unique case insensitive word in the user agent string identifying web crawlers that should not be allowed to access the site. Does not apply if allowlist is defined."+ crawler_asns: "ASNs operated by verified web crawlers. Traffic from these networks is excluded from browser pageview reporting and receives a high crawler score."+ crawler_detection_datacenter_asns: "ASNs that provide cloud, hosting, or data center services. These networks are not necessarily crawlers: their traffic receives a low-confidence crawler score and is still counted as browser traffic."+ crawler_stale_chromium_major_version_cutoff: "Chromium major versions at or below this value in unengaged sessions receive a crawler score. Update it only after reviewing traffic; an outdated cutoff becomes less sensitive, not more aggressive."slow_down_crawler_user_agents: "User agents of web crawlers that should be rate limited as configured in the {{setting:slow_down_crawler_rate}} setting. Each value must be at least 3 characters long."slow_down_crawler_rate: "If {{setting:slow_down_crawler_user_agents}} is specified this rate will apply to all the crawlers (number of seconds delay between requests)"llms_txt: "Upload a text file to be served at /llms.txt for LLM crawlers. See <a href='https://llmstxt.org/' target='_blank'>llmstxt.org</a> for more information."diff --git a/config/site_settings.yml b/config/site_settings.ymlindex 913d550c6d1..bb20202fc02 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -659,6 +659,12 @@ basic:crawler_automation_user_agents:default: "HeadlessChrome|Playwright|Puppeteer|Selenium|PhantomJS|jsdom"hidden: true+ crawler_stale_chromium_major_version_cutoff:+ default: 138+ type: integer+ min: 1+ max: 999+ hidden: trueinterface_color_selector:client: trueenum: "InterfaceColorSelectorSetting"@@ -3118,16 +3124,23 @@ security:crawler_check_bypass_agents:hidden: truedefault: "cubot|MetaIAB"- # 16509/14618=AWS, 396982=Google Cloud, 15169/36040=Google, 8075=Microsoft,- # 714=Apple, 13238=Yandex, 32934=Meta, 13335=Cloudflare,- # 140577=Ahrefs, 210743=Babbar, 7941=Internet Archive,- # 16276=OVH, 24940=Hetzner, 136907=Huawei Cloud, 51747=Internet Vikings,- # 4134/4837=China Telecom/Unicom, 5089=Virgin, 55836=Reliance Jio,- # 5410=Bouygues, 701=Verizon, 6805=Telefonica, 45899=VNPT- # 55967=Baidu+ # 38365/55967=Baidu, 140577=Ahrefs, 210743=Babbar, 7941=Internet Archive,+ # 13238=Yandexcrawler_asns:hidden: true- default: "16509|16276|714|13238|14618|136907|32934|15169|4134|4837|24940|396982|8075|51747|140577|45899|210743|13335|36040|5089|55836|5410|701|7941|6805|55967"+ default: "38365|55967|140577|210743|7941|13238"+ type: list+ list_type: compact+ # 16509/14618=AWS, 16276=OVH, 136907/55990=Huawei Cloud,+ # 24940=Hetzner, 396982=Google Cloud, 8075=Microsoft, 51747=Internet Vikings,+ # 13335=Cloudflare, 45090=Tencent Cloud, 23724=China Telecom IDC,+ # 37963=Alibaba Cloud, 55960=Sinnet, 203020=HostRoyale,+ # 63835=China Telecom Changsha IDC, 58540=China Telecom Jinan IDC,+ # 58541=China Telecom Qingdao IDC, 58563=China Telecom Hubei IDC,+ # 58461=China Telecom Hangzhou IDC+ crawler_detection_datacenter_asns:+ hidden: true+ default: "16509|16276|14618|136907|24940|396982|8075|51747|13335|45090|55990|23724|37963|55960|203020|63835|58540|58541|58563|58461"type: listlist_type: compactcors_origins:diff --git a/db/migrate/20260731055703_add_crawler_signals_to_browser_pageview_event_scores.rb b/db/migrate/20260731055703_add_crawler_signals_to_browser_pageview_event_scores.rbnew file mode 100644index 00000000000..9779c5b41a6--- /dev/null+++ b/db/migrate/20260731055703_add_crawler_signals_to_browser_pageview_event_scores.rb@@ -0,0 +1,21 @@+# frozen_string_literal: true++class AddCrawlerSignalsToBrowserPageviewEventScores < ActiveRecord::Migration[8.0]+ def change+ add_column :browser_pageview_event_scores,+ :datacenter_asn_score,+ :smallint,+ null: false,+ default: 0+ add_column :browser_pageview_event_scores,+ :single_request_no_referrer_score,+ :smallint,+ null: false,+ default: 0+ add_column :browser_pageview_event_scores,+ :stale_browser_score,+ :smallint,+ null: false,+ default: 0+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex f2925f6d9fa..e9c0d44918a 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -1992,7 +1992,10 @@ CREATE TABLE public.browser_pageview_event_scores (rapid_nav_score smallint DEFAULT 0 NOT NULL,referrer_score smallint DEFAULT 0 NOT NULL,engagement_score smallint DEFAULT 0 NOT NULL,- ip_rotation_score smallint DEFAULT 0 NOT NULL+ ip_rotation_score smallint DEFAULT 0 NOT NULL,+ datacenter_asn_score smallint DEFAULT 0 NOT NULL,+ single_request_no_referrer_score smallint DEFAULT 0 NOT NULL,+ stale_browser_score smallint DEFAULT 0 NOT NULL);@@ -23072,6 +23075,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groupsSET search_path TO "$user", public;INSERT INTO "schema_migrations" (version) VALUES+('20260731055703'),('20260729153343'),('20260728162521'),('20260728162516'),diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rbindex 0e3a3fcd97c..c49b8ce7f5c 100644--- a/lib/crawler_scorer.rb+++ b/lib/crawler_scorer.rb@@ -5,7 +5,11 @@ class CrawlerScorerAUTOMATION_UA_SCORE = 100- KNOWN_ASN_SCORE = 15+ KNOWN_ASN_SCORE = 30+ DATACENTER_ASN_SCORE = 10+ SINGLE_REQUEST_NO_REFERRER_SCORE = 10+ SINGLE_REQUEST_LOCALE_PARAM_BONUS = 5+ STALE_BROWSER_SCORE = 5VELOCITY_LOW = 150VELOCITY_MEDIUM = 300@@ -38,6 +42,8 @@ class CrawlerScorerdef self.score!(window_start:, window_end:)crawler_asns = SiteSetting.crawler_asns_map.map(&:to_i)+ crawler_detection_datacenter_asns =+ SiteSetting.crawler_detection_datacenter_asns_map.map(&:to_i)ActiveRecord::Base.transaction doDB.exec(@@ -46,9 +52,16 @@ class CrawlerScorerwindow_end: window_end,ua_regex: SiteSetting.crawler_automation_user_agents,crawler_asns: crawler_asns,+ crawler_detection_datacenter_asns: crawler_detection_datacenter_asns,hostname: Discourse.current_hostname,automation_ua_score: AUTOMATION_UA_SCORE,known_asn_score: KNOWN_ASN_SCORE,+ datacenter_asn_score: DATACENTER_ASN_SCORE,+ single_request_no_referrer_score: SINGLE_REQUEST_NO_REFERRER_SCORE,+ single_request_locale_param_bonus: SINGLE_REQUEST_LOCALE_PARAM_BONUS,+ stale_browser_score: STALE_BROWSER_SCORE,+ stale_chromium_major_version_cutoff:+ SiteSetting.crawler_stale_chromium_major_version_cutoff,velocity_low: VELOCITY_LOW,velocity_medium: VELOCITY_MEDIUM,velocity_high: VELOCITY_HIGH,@@ -78,7 +91,7 @@ class CrawlerScorerSQL = <<~SQLWITH events AS (- SELECT id, session_id, ip_address, user_agent, referrer, asn, created_at, source+ SELECT id, session_id, ip_address, user_agent, referrer, asn, url, created_at, sourceFROM browser_pageview_eventsWHERE created_at >= :window_startAND created_at < :window_end@@ -150,6 +163,30 @@ class CrawlerScorerWHEN e.asn = ANY(ARRAY[:crawler_asns]::int[]) THEN :known_asn_scoreELSE 0END AS known_asn_score,+ CASE+ WHEN e.asn = ANY(ARRAY[:crawler_detection_datacenter_asns]::int[]) THEN :datacenter_asn_score+ ELSE 0+ END AS datacenter_asn_score,+ CASE+ WHEN iu.pageviews = 1+ AND e.referrer IS NULL+ AND se.session_id IS NULL+ AND e.url ~ '[?&]#{Discourse::LOCALE_PARAM}(=|&|$)'+ THEN :single_request_no_referrer_score + :single_request_locale_param_bonus+ WHEN iu.pageviews = 1+ AND e.referrer IS NULL+ AND se.session_id IS NULL THEN :single_request_no_referrer_score+ ELSE 0+ END AS single_request_no_referrer_score,+ CASE+ WHEN se.session_id IS NULL+ AND e.user_agent ~ 'Chrome/[0-9]{1,3}'+ AND e.user_agent !~* 'HeadlessChrome'+ AND (substring(e.user_agent FROM 'Chrome/([0-9]{1,3})'))::int+ <= :stale_chromium_major_version_cutoff+ THEN :stale_browser_score+ ELSE 0+ END AS stale_browser_score,CASEWHEN iu.pageviews >= :velocity_high THEN :velocity_high_scoreWHEN iu.pageviews >= :velocity_medium THEN :velocity_medium_score@@ -205,6 +242,9 @@ class CrawlerScorerid,automation_ua_score,known_asn_score,+ datacenter_asn_score,+ single_request_no_referrer_score,+ stale_browser_score,velocity_score,churn_score,rapid_nav_score,@@ -213,11 +253,11 @@ class CrawlerScorerengagement_score,GREATEST(0,- automation_ua_score + known_asn_score + velocity_score + churn_score+ automation_ua_score + known_asn_score + datacenter_asn_score + single_request_no_referrer_score + stale_browser_score + velocity_score + churn_score+ rapid_nav_score + ip_rotation_score + referrer_score + engagement_score) AS scoreFROM breakdown- WHERE automation_ua_score + known_asn_score + velocity_score + churn_score+ WHERE automation_ua_score + known_asn_score + datacenter_asn_score + single_request_no_referrer_score + stale_browser_score + velocity_score + churn_score+ rapid_nav_score + ip_rotation_score + referrer_score > 0),@@ -230,6 +270,9 @@ class CrawlerScorerRETURNING e.id,t.automation_ua_score,t.known_asn_score,+ t.datacenter_asn_score,+ t.single_request_no_referrer_score,+ t.stale_browser_score,t.velocity_score,t.churn_score,t.rapid_nav_score,@@ -242,6 +285,9 @@ class CrawlerScorerevent_id,automation_ua_score,known_asn_score,+ datacenter_asn_score,+ single_request_no_referrer_score,+ stale_browser_score,velocity_score,churn_score,rapid_nav_score,@@ -253,6 +299,9 @@ class CrawlerScorerid,automation_ua_score,known_asn_score,+ datacenter_asn_score,+ single_request_no_referrer_score,+ stale_browser_score,velocity_score,churn_score,rapid_nav_score,@@ -263,6 +312,9 @@ class CrawlerScorerON CONFLICT (event_id) DO UPDATESET automation_ua_score = EXCLUDED.automation_ua_score,known_asn_score = EXCLUDED.known_asn_score,+ datacenter_asn_score = EXCLUDED.datacenter_asn_score,+ single_request_no_referrer_score = EXCLUDED.single_request_no_referrer_score,+ stale_browser_score = EXCLUDED.stale_browser_score,velocity_score = EXCLUDED.velocity_score,churn_score = EXCLUDED.churn_score,rapid_nav_score = EXCLUDED.rapid_nav_score,diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rbindex a65f0e12943..c62705849b4 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb@@ -1541,6 +1541,9 @@ module DiscourseDataExplorerCOUNT(*) AS pageviews,MAX(s.automation_ua_score) AS automation_ua,MAX(s.known_asn_score) AS known_asn,+ MAX(s.datacenter_asn_score) AS datacenter_asn,+ MAX(s.single_request_no_referrer_score) AS single_request_no_referrer,+ MAX(s.stale_browser_score) AS stale_browser,MAX(s.velocity_score) AS velocity,MAX(s.churn_score) AS churn,MAX(s.rapid_nav_score) AS rapid_nav,@@ -1551,6 +1554,14 @@ module DiscourseDataExplorerCONCAT_WS(', ',CASE WHEN MAX(s.automation_ua_score) > 0 THEN 'automation UA' END,CASE WHEN MAX(s.known_asn_score) > 0 THEN 'known crawler ASN' END,+ CASE WHEN MAX(s.datacenter_asn_score) > 0 THEN 'datacenter ASN' END,+ CASE+ WHEN MAX(s.single_request_no_referrer_score) = #{CrawlerScorer::SINGLE_REQUEST_NO_REFERRER_SCORE + CrawlerScorer::SINGLE_REQUEST_LOCALE_PARAM_BONUS}+ THEN 'single direct locale request (+' || MAX(s.single_request_no_referrer_score) || ')'+ WHEN MAX(s.single_request_no_referrer_score) > 0+ THEN 'single direct request (+' || MAX(s.single_request_no_referrer_score) || ')'+ END,+ CASE WHEN MAX(s.stale_browser_score) > 0 THEN 'stale Chromium (+' || MAX(s.stale_browser_score) || ')' END,CASE WHEN MAX(s.velocity_score) > 0 THEN 'high velocity (+' || MAX(s.velocity_score) || ')' END,CASE WHEN MAX(s.churn_score) > 0 THEN 'session churn (+' || MAX(s.churn_score) || ')' END,CASE WHEN MAX(s.rapid_nav_score) > 0 THEN 'rapid navigation' END,diff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rbindex 7536030eaae..c5319871ca2 100644--- a/spec/lib/crawler_scorer_spec.rb+++ b/spec/lib/crawler_scorer_spec.rb@@ -35,10 +35,13 @@ RSpec.describe CrawlerScorer doscore!- expect(event.reload.score).to eq(155)+ expect(event.reload.score).to eq(170)breakdown = event.browser_pageview_event_scoreexpect(breakdown.automation_ua_score).to eq(100)- expect(breakdown.known_asn_score).to eq(15)+ expect(breakdown.known_asn_score).to eq(30)+ expect(breakdown.datacenter_asn_score).to eq(0)+ expect(breakdown.single_request_no_referrer_score).to eq(0)+ expect(breakdown.stale_browser_score).to eq(0)expect(breakdown.velocity_score).to eq(0)expect(breakdown.churn_score).to eq(0)expect(breakdown.rapid_nav_score).to eq(0)@@ -49,16 +52,103 @@ RSpec.describe CrawlerScorer doit "does not score an event whose only signal is missing engagement" doevent = make_event+score!+expect(event.reload.score).to be_nilexpect(event.browser_pageview_event_score).to be_nilend- it "scores known crawler ASNs at +15" do+ it "scores known crawler ASNs at +30" doSiteSetting.crawler_asns = "12345"event = make_event(asn: 12_345)score!+ expect(event.reload.score).to eq(70)+ end++ it "scores datacenter ASNs at +10" do+ SiteSetting.crawler_asns = ""+ SiteSetting.crawler_detection_datacenter_asns = "12345"+ event = make_event(asn: 12_345)++ score!++ expect(event.reload.score).to eq(50)+ expect(event.browser_pageview_event_score.datacenter_asn_score).to eq(10)+ end++ it "scores unengaged single direct requests at +10" do+ event = make_event(referrer: nil)++ score!++ expect(event.reload.score).to eq(50)+ expect(event.browser_pageview_event_score.single_request_no_referrer_score).to eq(10)+ end++ it "scores unengaged single direct requests with any translation parameter at +15" do+ event = make_event(url: "/t/topic/1?source=bot&tl=made-up-locale", referrer: nil)++ score!+expect(event.reload.score).to eq(55)+ expect(event.browser_pageview_event_score.single_request_no_referrer_score).to eq(15)+ end++ it "scores unengaged Chromium versions at or below the configured cutoff at +5" do+ SiteSetting.crawler_stale_chromium_major_version_cutoff = 138+ event = make_event(user_agent: "Mozilla/5.0 Chrome/138.0.0.0 Safari/537.36")++ score!++ expect(event.reload.score).to eq(45)+ expect(event.browser_pageview_event_score.stale_browser_score).to eq(5)+ end++ it "does not score Chromium versions above the configured cutoff" do+ SiteSetting.crawler_stale_chromium_major_version_cutoff = 138+ event = make_event(user_agent: "Mozilla/5.0 Chrome/139.0.0.0 Safari/537.36")++ score!++ expect(event.reload.score).to be_nil+ expect(event.browser_pageview_event_score).to be_nil+ end++ it "does not score oversized Chromium major versions as stale" do+ event = make_event(user_agent: "Mozilla/5.0 Chrome/99999999999.0.0.0 Safari/537.36")++ score!++ expect(event.reload.score).to be_nil+ expect(event.browser_pageview_event_score).to be_nil+ end++ it "does not score stale Chromium versions when the session has interaction" do+ SiteSetting.crawler_asns = "12345"+ event = make_event(asn: 12_345, user_agent: "Mozilla/5.0 Chrome/125.0.0.0 Safari/537.36")+ Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, click_events: 1)++ score!++ expect(event.reload.score).to eq(30)+ expect(event.browser_pageview_event_score.stale_browser_score).to eq(0)+ end++ it "scores stale Chromium versions reported by Edge and Opera" do+ SiteSetting.crawler_stale_chromium_major_version_cutoff = 138+ edge_event = make_event(user_agent: "Mozilla/5.0 Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0")+ opera_event = make_event(user_agent: "Mozilla/5.0 Chrome/125.0.0.0 Safari/537.36 OPR/110.0.0.0")++ score!++ expect([edge_event.reload.score, opera_event.reload.score]).to eq([45, 45])+ expect(+ [+ edge_event.browser_pageview_event_score.stale_browser_score,+ opera_event.browser_pageview_event_score.stale_browser_score,+ ],+ ).to eq([5, 5])endit "scores pageview velocity at or above VELOCITY_LOW threshold at +15" do@@ -138,7 +228,7 @@ RSpec.describe CrawlerScorer do).to contain_exactly(70)end- it "does not score a session that only ever uses two addresses" do+ it "does not add an IP rotation score to a session that only uses two addresses" dobase = 30.minutes.ago%w[10.0.0.1 10.0.0.2].each_with_index do |ip, i|make_event(ip_address: ip, session_id: "handover-session", created_at: base + (i * 5).seconds)@@ -151,7 +241,7 @@ RSpec.describe CrawlerScorer do)end- it "does not score ip changes slow enough to be a network handover" do+ it "does not add an IP rotation score to a slow network handover" dobase = 40.minutes.ago%w[10.0.0.1 10.0.0.2 10.0.0.3].each_with_index do |ip, i|make_event(ip_address: ip, session_id: "slow-session", created_at: base + (i * 11).minutes)@@ -221,9 +311,10 @@ RSpec.describe CrawlerScorer doscore!- expect(event.reload.score).to eq(15)+ expect(event.reload.score).to eq(30)breakdown = event.browser_pageview_event_score- expect(breakdown.known_asn_score).to eq(15)+ expect(breakdown.known_asn_score).to eq(30)+ expect(breakdown.datacenter_asn_score).to eq(0)expect(breakdown.engagement_score).to eq(0)end@@ -246,12 +337,12 @@ RSpec.describe CrawlerScorer doevent = make_event(asn: 12_345)score!- expect(event.reload.score).to eq(55)+ expect(event.reload.score).to eq(70)Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, key_events: 4)score!- expect(event.reload.score).to eq(55)+ expect(event.reload.score).to eq(70)endit "scores each source but partitions velocity so transports do not inflate each other" do