Skip to content

A platform for community discussion. Free, open, simple.

DEV: Add anonymous pageview crawler scoring job (#39954)

Krzysztof Kotlarek9b2c7beadf2e
diff --git a/app/jobs/scheduled/detect_crawler_pageviews.rb b/app/jobs/scheduled/detect_crawler_pageviews.rb
new file mode 100644
index 00000000000..1e27489714f
--- /dev/null
+++ b/app/jobs/scheduled/detect_crawler_pageviews.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Jobs
+ class DetectCrawlerPageviews < ::Jobs::Scheduled
+ every 10.minutes
+
+ LOOKBACK = 1.hour
+
+ def execute(args)
+ return if !SiteSetting.experimental_detect_crawler_pageviews
+
+ now = Time.now
+ CrawlerScorer.score!(window_start: now - LOOKBACK, window_end: now)
+ end
+ end
+end
diff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rb
index d4db91e857b..7ae01d2fa5f 100644
--- a/app/models/browser_pageview_event.rb
+++ b/app/models/browser_pageview_event.rb
@@ -23,9 +23,11 @@ end
# Table name: browser_pageview_events
#
# id :bigint not null, primary key
+# asn :integer
# country_code :string(2)
# ip_address :inet not null
# referrer :string(2000)
+# score :integer
# url :string(2000) not null
# user_agent :string(1000) not null
# created_at :datetime not null
@@ -35,6 +37,8 @@ end
#
# Indexes
#
+# idx_bpe_ip_ua_created_at (ip_address,user_agent,created_at)
+# idx_bpe_session_created_at (session_id,created_at)
# index_browser_pageview_events_on_created_at (created_at) USING brin
# index_browser_pageview_events_on_topic_id (topic_id)
# index_browser_pageview_events_on_user_id (user_id)
diff --git a/config/site_settings.yml b/config/site_settings.yml
index 0c5d689157a..4b6ec478e80 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -615,6 +615,12 @@ basic:
clean_up_browser_pageview_events:
default: false
hidden: true
+ experimental_detect_crawler_pageviews:
+ default: false
+ hidden: true
+ crawler_automation_user_agents:
+ default: "HeadlessChrome|Playwright|Puppeteer|Selenium|PhantomJS|jsdom"
+ hidden: true
interface_color_selector:
client: true
enum: "InterfaceColorSelectorSetting"
@@ -2877,9 +2883,15 @@ security:
crawler_check_bypass_agents:
hidden: true
default: "cubot|MetaIAB"
+ # 16509/14618=AWS, 396982=Google Cloud, 15169/36040=Google, 8075=Microsoft,
+ # 714=Apple, 13238=Yandex, 32934=Meta, 13335=Cloudflare,
+ # 140577=Ahrefs, 210743=Babbar, 7941=Internet Archive,
+ # 16276=OVH, 24940=Hetzner, 136907=Huawei Cloud, 51747=Internet Vikings,
+ # 4134/4837=China Telecom/Unicom, 5089=Virgin, 55836=Reliance Jio,
+ # 5410=Bouygues, 701=Verizon, 6805=Telefonica, 45899=VNPT
crawler_asns:
hidden: true
- default: "55967"
+ default: "16509|16276|714|13238|14618|136907|32934|15169|4134|4837|24940|396982|8075|51747|140577|45899|210743|13335|36040|5089|55836|5410|701|7941|6805"
type: list
list_type: compact
cors_origins:
diff --git a/db/migrate/20260513024004_add_browser_pageview_event_scores_and_asn.rb b/db/migrate/20260513024004_add_browser_pageview_event_scores_and_asn.rb
new file mode 100644
index 00000000000..6b4c6444332
--- /dev/null
+++ b/db/migrate/20260513024004_add_browser_pageview_event_scores_and_asn.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class AddBrowserPageviewEventScoresAndAsn < ActiveRecord::Migration[8.0]
+ def change
+ add_column :browser_pageview_events, :asn, :integer, null: true
+ add_column :browser_pageview_events, :score, :integer, null: true
+
+ add_index :browser_pageview_events,
+ %i[session_id created_at],
+ name: "idx_bpe_session_created_at"
+
+ add_index :browser_pageview_events,
+ %i[ip_address user_agent created_at],
+ name: "idx_bpe_ip_ua_created_at"
+ end
+end
diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rb
new file mode 100644
index 00000000000..ec77a80072b
--- /dev/null
+++ b/lib/crawler_scorer.rb
@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+class CrawlerScorer
+ AUTOMATION_UA_SCORE = 100
+
+ KNOWN_ASN_SCORE = 15
+
+ VELOCITY_LOW = 120
+ VELOCITY_MEDIUM = 300
+ VELOCITY_HIGH = 600
+ VELOCITY_LOW_SCORE = 10
+ VELOCITY_MEDIUM_SCORE = 20
+ VELOCITY_HIGH_SCORE = 35
+
+ CHURN_LOW_MIN_SESSIONS = 5
+ CHURN_HIGH_MIN_SESSIONS = 10
+ CHURN_MAX_AVG_EVENTS = 2
+ CHURN_LOW_SCORE = 10
+ CHURN_HIGH_SCORE = 20
+
+ RAPID_NAV_MIN_GAPS = 10
+ RAPID_NAV_MAX_MEDIAN_SECONDS = 5
+ RAPID_NAV_SCORE = 15
+
+ REFERRER_MIN_EVENTS = 5
+ REFERRER_LOW_RATIO = 0.5
+ REFERRER_HIGH_RATIO = 0.8
+ REFERRER_LOW_SCORE = 5
+ REFERRER_HIGH_SCORE = 10
+
+ def self.score!(window_start:, window_end:)
+ crawler_asns = SiteSetting.crawler_asns_map.map(&:to_i)
+
+ ActiveRecord::Base.transaction do
+ DB.exec(
+ SQL,
+ window_start: window_start,
+ window_end: window_end,
+ ua_regex: SiteSetting.crawler_automation_user_agents,
+ crawler_asns: crawler_asns,
+ hostname: Discourse.current_hostname,
+ automation_ua_score: AUTOMATION_UA_SCORE,
+ known_asn_score: KNOWN_ASN_SCORE,
+ velocity_low: VELOCITY_LOW,
+ velocity_medium: VELOCITY_MEDIUM,
+ velocity_high: VELOCITY_HIGH,
+ velocity_low_score: VELOCITY_LOW_SCORE,
+ velocity_medium_score: VELOCITY_MEDIUM_SCORE,
+ velocity_high_score: VELOCITY_HIGH_SCORE,
+ churn_low_min_sessions: CHURN_LOW_MIN_SESSIONS,
+ churn_high_min_sessions: CHURN_HIGH_MIN_SESSIONS,
+ churn_max_avg_events: CHURN_MAX_AVG_EVENTS,
+ churn_low_score: CHURN_LOW_SCORE,
+ churn_high_score: CHURN_HIGH_SCORE,
+ rapid_nav_min_gaps: RAPID_NAV_MIN_GAPS,
+ rapid_nav_max_median_seconds: RAPID_NAV_MAX_MEDIAN_SECONDS,
+ rapid_nav_score: RAPID_NAV_SCORE,
+ referrer_min_events: REFERRER_MIN_EVENTS,
+ referrer_low_ratio: REFERRER_LOW_RATIO,
+ referrer_high_ratio: REFERRER_HIGH_RATIO,
+ referrer_low_score: REFERRER_LOW_SCORE,
+ referrer_high_score: REFERRER_HIGH_SCORE,
+ )
+ end
+ end
+
+ SQL = <<~SQL
+ WITH events AS (
+ SELECT id, session_id, ip_address, user_agent, referrer, asn, created_at
+ FROM browser_pageview_events
+ WHERE created_at >= :window_start
+ AND created_at < :window_end
+ ),
+
+ ipua_stats AS (
+ SELECT
+ ip_address,
+ user_agent,
+ COUNT(*) AS pageviews,
+ COUNT(DISTINCT session_id) AS distinct_sessions,
+ AVG(
+ CASE
+ WHEN referrer IS NULL THEN 1.0
+ WHEN substring(referrer from '^https?://([^/]+)') = :hostname THEN 0.0
+ ELSE 1.0
+ END
+ ) AS bad_referrer_ratio
+ FROM events
+ GROUP BY ip_address, user_agent
+ ),
+
+ gaps AS (
+ SELECT
+ ip_address,
+ user_agent,
+ EXTRACT(EPOCH FROM created_at - LAG(created_at) OVER (
+ PARTITION BY ip_address, user_agent ORDER BY created_at
+ )) AS gap_seconds
+ FROM events
+ ),
+
+ median_gap AS (
+ SELECT
+ ip_address,
+ user_agent,
+ percentile_cont(0.5) WITHIN GROUP (ORDER BY gap_seconds)
+ AS median_gap_seconds,
+ COUNT(*) AS gap_count
+ FROM gaps
+ WHERE gap_seconds IS NOT NULL
+ GROUP BY ip_address, user_agent
+ ),
+
+ totals AS (
+ SELECT
+ e.id,
+ CASE
+ WHEN :ua_regex <> '' AND e.user_agent ~* :ua_regex THEN :automation_ua_score
+ ELSE 0
+ END
+ + CASE
+ WHEN e.asn = ANY(ARRAY[:crawler_asns]::int[]) THEN :known_asn_score
+ ELSE 0
+ END
+ + CASE
+ WHEN iu.pageviews >= :velocity_high THEN :velocity_high_score
+ WHEN iu.pageviews >= :velocity_medium THEN :velocity_medium_score
+ WHEN iu.pageviews >= :velocity_low THEN :velocity_low_score
+ ELSE 0
+ END
+ + CASE
+ WHEN iu.distinct_sessions >= :churn_high_min_sessions
+ AND iu.pageviews::float / NULLIF(iu.distinct_sessions, 0) <= :churn_max_avg_events
+ THEN :churn_high_score
+ WHEN iu.distinct_sessions >= :churn_low_min_sessions
+ AND iu.pageviews::float / NULLIF(iu.distinct_sessions, 0) <= :churn_max_avg_events
+ THEN :churn_low_score
+ ELSE 0
+ END
+ + CASE
+ WHEN mg.gap_count >= :rapid_nav_min_gaps
+ AND mg.median_gap_seconds < :rapid_nav_max_median_seconds
+ THEN :rapid_nav_score
+ ELSE 0
+ END
+ + CASE
+ WHEN iu.pageviews >= :referrer_min_events
+ AND iu.bad_referrer_ratio >= :referrer_high_ratio THEN :referrer_high_score
+ WHEN iu.pageviews >= :referrer_min_events
+ AND iu.bad_referrer_ratio >= :referrer_low_ratio THEN :referrer_low_score
+ ELSE 0
+ END
+ AS score
+ FROM events e
+ LEFT JOIN ipua_stats iu USING (ip_address, user_agent)
+ LEFT JOIN median_gap mg USING (ip_address, user_agent)
+ )
+
+ UPDATE browser_pageview_events e
+ SET score = t.score
+ FROM totals t
+ WHERE e.id = t.id
+ AND t.score > COALESCE(e.score, 0);
+ SQL
+end
diff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rb
index 7e4b3a76d87..8c72a394f0d 100644
--- a/lib/middleware/request_tracker.rb
+++ b/lib/middleware/request_tracker.rb
@@ -723,6 +723,7 @@ class Middleware::RequestTracker
url: payload[:url],
ip_address: payload[:ip_address],
country_code: payload[:country_code],
+ asn: payload[:asn],
referrer: payload[:referrer],
user_agent: payload[:user_agent],
session_id: payload[:session_id],
@@ -749,11 +750,13 @@ class Middleware::RequestTracker
private_class_method :trigger_beacon_browser_pageview_event
def self.build_browser_pageview_event_payload(data)
+ ip_info = DiscourseIpInfo.get(data[:request_remote_ip])
{
user_id: data[:current_user_id],
url: data[:tracking_url],
ip_address: data[:request_remote_ip],
- country_code: DiscourseIpInfo.get(data[:request_remote_ip])[:country_code],
+ country_code: ip_info[:country_code],
+ asn: ip_info[:asn],
user_agent: data[:user_agent],
referrer: data[:tracking_referrer],
session_id: data[:tracking_session_id],
diff --git a/spec/jobs/detect_crawler_pageviews_spec.rb b/spec/jobs/detect_crawler_pageviews_spec.rb
new file mode 100644
index 00000000000..defc915fc8f
--- /dev/null
+++ b/spec/jobs/detect_crawler_pageviews_spec.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+RSpec.describe Jobs::DetectCrawlerPageviews do
+ it "does nothing when detection is disabled" do
+ SiteSetting.experimental_detect_crawler_pageviews = false
+ CrawlerScorer.expects(:score!).never
+
+ described_class.new.execute({})
+ end
+
+ it "scores the last hour of pageviews when enabled" do
+ SiteSetting.experimental_detect_crawler_pageviews = true
+ freeze_time
+
+ CrawlerScorer.expects(:score!).with(window_start: 1.hour.ago, window_end: Time.now)
+
+ described_class.new.execute({})
+ end
+end
diff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rb
new file mode 100644
index 00000000000..04a1cb4cab4
--- /dev/null
+++ b/spec/lib/crawler_scorer_spec.rb
@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+RSpec.describe CrawlerScorer do
+ let(:hostname) { Discourse.current_hostname }
+
+ def make_event(opts = {})
+ defaults = {
+ url: "/t/topic/1",
+ ip_address: "1.2.3.4",
+ user_agent: "Mozilla/5.0",
+ session_id: SecureRandom.hex(8),
+ created_at: 30.minutes.ago,
+ referrer: "https://#{hostname}/",
+ }
+ BrowserPageviewEvent.create!(defaults.merge(opts))
+ end
+
+ def score!
+ described_class.score!(window_start: 1.hour.ago, window_end: Time.now)
+ end
+
+ it "scores automation user agents at +100" do
+ event = make_event(user_agent: "Mozilla/5.0 (X11; Linux x86_64) HeadlessChrome/120.0.0.0")
+ score!
+ expect(event.reload.score).to eq(100)
+ end
+
+ it "scores known crawler ASNs at +15" do
+ SiteSetting.crawler_asns = "12345"
+ event = make_event(asn: 12_345)
+ score!
+ expect(event.reload.score).to eq(15)
+ end
+
+ it "scores pageview velocity at or above VELOCITY_LOW threshold at +10" do
+ stub_const(CrawlerScorer, :VELOCITY_LOW, 10) do
+ session_id = "burst-session"
+ base = 30.minutes.ago
+ 10.times { |i| make_event(session_id: session_id, created_at: base + (i * 15).seconds) }
+
+ score!
+
+ expect(
+ BrowserPageviewEvent.where(session_id: session_id).pluck(:score).uniq,
+ ).to contain_exactly(10)
+ end
+ end
+
+ it "scores session churn when one ip+ua spawns many short sessions" do
+ stub_const(CrawlerScorer, :CHURN_HIGH_MIN_SESSIONS, 3) do
+ 3.times do |i|
+ make_event(ip_address: "9.9.9.9", user_agent: "ScriptyBot/1.0", session_id: "churn-#{i}")
+ end
+
+ score!
+
+ expect(
+ BrowserPageviewEvent.where(ip_address: "9.9.9.9").pluck(:score).uniq,
+ ).to contain_exactly(20)
+ end
+ end
+
+ it "scores rapid navigation when the median gap is under 5 seconds" do
+ stub_const(CrawlerScorer, :RAPID_NAV_MIN_GAPS, 3) do
+ session_id = "rapid-session"
+ base = 30.minutes.ago
+ 4.times { |i| make_event(session_id: session_id, created_at: base + i.seconds) }
+
+ score!
+
+ expect(
+ BrowserPageviewEvent.where(session_id: session_id).pluck(:score).uniq,
+ ).to contain_exactly(15)
+ end
+ end
+
+ it "scores referrer discontinuity when most pageviews have no referrer" do
+ stub_const(CrawlerScorer, :REFERRER_MIN_EVENTS, 2) do
+ session_id = "ref-session"
+ 2.times do
+ make_event(
+ ip_address: "5.5.5.5",
+ user_agent: "RefBot/1.0",
+ session_id: session_id,
+ referrer: nil,
+ )
+ end
+
+ score!
+
+ expect(
+ BrowserPageviewEvent.where(ip_address: "5.5.5.5").pluck(:score).uniq,
+ ).to contain_exactly(10)
+ end
+ end
+
+ it "also scores logged-in events" do
+ user = Fabricate(:user)
+ event = make_event(user_id: user.id, user_agent: "Mozilla/5.0 HeadlessChrome/120.0.0.0")
+
+ score!
+
+ expect(event.reload.score).to eq(100)
+ end
+
+ it "ignores events outside the window" do
+ event = make_event(user_agent: "HeadlessChrome/120", created_at: 2.hours.ago)
+
+ score!
+
+ expect(event.reload.score).to be_nil
+ end
+
+ it "only updates the score when the new value is higher" do
+ event = make_event(user_agent: "Mozilla/5.0 HeadlessChrome/120")
+ event.update!(score: 120)
+
+ score!
+
+ expect(event.reload.score).to eq(120)
+ end
+end