A platform for community discussion. Free, open, simple.
DEV: Add anonymous pageview crawler scoring job (#39954)
Krzysztof Kotlarek9b2c7beadf2e
diff --git a/app/jobs/scheduled/detect_crawler_pageviews.rb b/app/jobs/scheduled/detect_crawler_pageviews.rbnew file mode 100644index 00000000000..1e27489714f--- /dev/null+++ b/app/jobs/scheduled/detect_crawler_pageviews.rb@@ -0,0 +1,16 @@+# frozen_string_literal: true++module Jobs+ class DetectCrawlerPageviews < ::Jobs::Scheduled+ every 10.minutes++ LOOKBACK = 1.hour++ def execute(args)+ return if !SiteSetting.experimental_detect_crawler_pageviews++ now = Time.now+ CrawlerScorer.score!(window_start: now - LOOKBACK, window_end: now)+ end+ end+enddiff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rbindex d4db91e857b..7ae01d2fa5f 100644--- a/app/models/browser_pageview_event.rb+++ b/app/models/browser_pageview_event.rb@@ -23,9 +23,11 @@ end# Table name: browser_pageview_events## id :bigint not null, primary key+# asn :integer# country_code :string(2)# ip_address :inet not null# referrer :string(2000)+# score :integer# url :string(2000) not null# user_agent :string(1000) not null# created_at :datetime not null@@ -35,6 +37,8 @@ end## Indexes#+# idx_bpe_ip_ua_created_at (ip_address,user_agent,created_at)+# idx_bpe_session_created_at (session_id,created_at)# index_browser_pageview_events_on_created_at (created_at) USING brin# index_browser_pageview_events_on_topic_id (topic_id)# index_browser_pageview_events_on_user_id (user_id)diff --git a/config/site_settings.yml b/config/site_settings.ymlindex 0c5d689157a..4b6ec478e80 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -615,6 +615,12 @@ basic:clean_up_browser_pageview_events:default: falsehidden: true+ experimental_detect_crawler_pageviews:+ default: false+ hidden: true+ crawler_automation_user_agents:+ default: "HeadlessChrome|Playwright|Puppeteer|Selenium|PhantomJS|jsdom"+ hidden: trueinterface_color_selector:client: trueenum: "InterfaceColorSelectorSetting"@@ -2877,9 +2883,15 @@ security:crawler_check_bypass_agents:hidden: truedefault: "cubot|MetaIAB"+ # 16509/14618=AWS, 396982=Google Cloud, 15169/36040=Google, 8075=Microsoft,+ # 714=Apple, 13238=Yandex, 32934=Meta, 13335=Cloudflare,+ # 140577=Ahrefs, 210743=Babbar, 7941=Internet Archive,+ # 16276=OVH, 24940=Hetzner, 136907=Huawei Cloud, 51747=Internet Vikings,+ # 4134/4837=China Telecom/Unicom, 5089=Virgin, 55836=Reliance Jio,+ # 5410=Bouygues, 701=Verizon, 6805=Telefonica, 45899=VNPTcrawler_asns:hidden: true- default: "55967"+ default: "16509|16276|714|13238|14618|136907|32934|15169|4134|4837|24940|396982|8075|51747|140577|45899|210743|13335|36040|5089|55836|5410|701|7941|6805"type: listlist_type: compactcors_origins:diff --git a/db/migrate/20260513024004_add_browser_pageview_event_scores_and_asn.rb b/db/migrate/20260513024004_add_browser_pageview_event_scores_and_asn.rbnew file mode 100644index 00000000000..6b4c6444332--- /dev/null+++ b/db/migrate/20260513024004_add_browser_pageview_event_scores_and_asn.rb@@ -0,0 +1,16 @@+# frozen_string_literal: true++class AddBrowserPageviewEventScoresAndAsn < ActiveRecord::Migration[8.0]+ def change+ add_column :browser_pageview_events, :asn, :integer, null: true+ add_column :browser_pageview_events, :score, :integer, null: true++ add_index :browser_pageview_events,+ %i[session_id created_at],+ name: "idx_bpe_session_created_at"++ add_index :browser_pageview_events,+ %i[ip_address user_agent created_at],+ name: "idx_bpe_ip_ua_created_at"+ end+enddiff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rbnew file mode 100644index 00000000000..ec77a80072b--- /dev/null+++ b/lib/crawler_scorer.rb@@ -0,0 +1,165 @@+# frozen_string_literal: true++class CrawlerScorer+ AUTOMATION_UA_SCORE = 100++ KNOWN_ASN_SCORE = 15++ VELOCITY_LOW = 120+ VELOCITY_MEDIUM = 300+ VELOCITY_HIGH = 600+ VELOCITY_LOW_SCORE = 10+ VELOCITY_MEDIUM_SCORE = 20+ VELOCITY_HIGH_SCORE = 35++ CHURN_LOW_MIN_SESSIONS = 5+ CHURN_HIGH_MIN_SESSIONS = 10+ CHURN_MAX_AVG_EVENTS = 2+ CHURN_LOW_SCORE = 10+ CHURN_HIGH_SCORE = 20++ RAPID_NAV_MIN_GAPS = 10+ RAPID_NAV_MAX_MEDIAN_SECONDS = 5+ RAPID_NAV_SCORE = 15++ REFERRER_MIN_EVENTS = 5+ REFERRER_LOW_RATIO = 0.5+ REFERRER_HIGH_RATIO = 0.8+ REFERRER_LOW_SCORE = 5+ REFERRER_HIGH_SCORE = 10++ def self.score!(window_start:, window_end:)+ crawler_asns = SiteSetting.crawler_asns_map.map(&:to_i)++ ActiveRecord::Base.transaction do+ DB.exec(+ SQL,+ window_start: window_start,+ window_end: window_end,+ ua_regex: SiteSetting.crawler_automation_user_agents,+ crawler_asns: crawler_asns,+ hostname: Discourse.current_hostname,+ automation_ua_score: AUTOMATION_UA_SCORE,+ known_asn_score: KNOWN_ASN_SCORE,+ velocity_low: VELOCITY_LOW,+ velocity_medium: VELOCITY_MEDIUM,+ velocity_high: VELOCITY_HIGH,+ velocity_low_score: VELOCITY_LOW_SCORE,+ velocity_medium_score: VELOCITY_MEDIUM_SCORE,+ velocity_high_score: VELOCITY_HIGH_SCORE,+ churn_low_min_sessions: CHURN_LOW_MIN_SESSIONS,+ churn_high_min_sessions: CHURN_HIGH_MIN_SESSIONS,+ churn_max_avg_events: CHURN_MAX_AVG_EVENTS,+ churn_low_score: CHURN_LOW_SCORE,+ churn_high_score: CHURN_HIGH_SCORE,+ rapid_nav_min_gaps: RAPID_NAV_MIN_GAPS,+ rapid_nav_max_median_seconds: RAPID_NAV_MAX_MEDIAN_SECONDS,+ rapid_nav_score: RAPID_NAV_SCORE,+ referrer_min_events: REFERRER_MIN_EVENTS,+ referrer_low_ratio: REFERRER_LOW_RATIO,+ referrer_high_ratio: REFERRER_HIGH_RATIO,+ referrer_low_score: REFERRER_LOW_SCORE,+ referrer_high_score: REFERRER_HIGH_SCORE,+ )+ end+ end++ SQL = <<~SQL+ WITH events AS (+ SELECT id, session_id, ip_address, user_agent, referrer, asn, created_at+ FROM browser_pageview_events+ WHERE created_at >= :window_start+ AND created_at < :window_end+ ),++ ipua_stats AS (+ SELECT+ ip_address,+ user_agent,+ COUNT(*) AS pageviews,+ COUNT(DISTINCT session_id) AS distinct_sessions,+ AVG(+ CASE+ WHEN referrer IS NULL THEN 1.0+ WHEN substring(referrer from '^https?://([^/]+)') = :hostname THEN 0.0+ ELSE 1.0+ END+ ) AS bad_referrer_ratio+ FROM events+ GROUP BY ip_address, user_agent+ ),++ gaps AS (+ SELECT+ ip_address,+ user_agent,+ EXTRACT(EPOCH FROM created_at - LAG(created_at) OVER (+ PARTITION BY ip_address, user_agent ORDER BY created_at+ )) AS gap_seconds+ FROM events+ ),++ median_gap AS (+ SELECT+ ip_address,+ user_agent,+ percentile_cont(0.5) WITHIN GROUP (ORDER BY gap_seconds)+ AS median_gap_seconds,+ COUNT(*) AS gap_count+ FROM gaps+ WHERE gap_seconds IS NOT NULL+ GROUP BY ip_address, user_agent+ ),++ totals AS (+ SELECT+ e.id,+ CASE+ WHEN :ua_regex <> '' AND e.user_agent ~* :ua_regex THEN :automation_ua_score+ ELSE 0+ END+ + CASE+ WHEN e.asn = ANY(ARRAY[:crawler_asns]::int[]) THEN :known_asn_score+ ELSE 0+ END+ + CASE+ WHEN iu.pageviews >= :velocity_high THEN :velocity_high_score+ WHEN iu.pageviews >= :velocity_medium THEN :velocity_medium_score+ WHEN iu.pageviews >= :velocity_low THEN :velocity_low_score+ ELSE 0+ END+ + CASE+ WHEN iu.distinct_sessions >= :churn_high_min_sessions+ AND iu.pageviews::float / NULLIF(iu.distinct_sessions, 0) <= :churn_max_avg_events+ THEN :churn_high_score+ WHEN iu.distinct_sessions >= :churn_low_min_sessions+ AND iu.pageviews::float / NULLIF(iu.distinct_sessions, 0) <= :churn_max_avg_events+ THEN :churn_low_score+ ELSE 0+ END+ + CASE+ WHEN mg.gap_count >= :rapid_nav_min_gaps+ AND mg.median_gap_seconds < :rapid_nav_max_median_seconds+ THEN :rapid_nav_score+ ELSE 0+ END+ + CASE+ WHEN iu.pageviews >= :referrer_min_events+ AND iu.bad_referrer_ratio >= :referrer_high_ratio THEN :referrer_high_score+ WHEN iu.pageviews >= :referrer_min_events+ AND iu.bad_referrer_ratio >= :referrer_low_ratio THEN :referrer_low_score+ ELSE 0+ END+ AS score+ FROM events e+ LEFT JOIN ipua_stats iu USING (ip_address, user_agent)+ LEFT JOIN median_gap mg USING (ip_address, user_agent)+ )++ UPDATE browser_pageview_events e+ SET score = t.score+ FROM totals t+ WHERE e.id = t.id+ AND t.score > COALESCE(e.score, 0);+ SQL+enddiff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rbindex 7e4b3a76d87..8c72a394f0d 100644--- a/lib/middleware/request_tracker.rb+++ b/lib/middleware/request_tracker.rb@@ -723,6 +723,7 @@ class Middleware::RequestTrackerurl: payload[:url],ip_address: payload[:ip_address],country_code: payload[:country_code],+ asn: payload[:asn],referrer: payload[:referrer],user_agent: payload[:user_agent],session_id: payload[:session_id],@@ -749,11 +750,13 @@ class Middleware::RequestTrackerprivate_class_method :trigger_beacon_browser_pageview_eventdef self.build_browser_pageview_event_payload(data)+ ip_info = DiscourseIpInfo.get(data[:request_remote_ip]){user_id: data[:current_user_id],url: data[:tracking_url],ip_address: data[:request_remote_ip],- country_code: DiscourseIpInfo.get(data[:request_remote_ip])[:country_code],+ country_code: ip_info[:country_code],+ asn: ip_info[:asn],user_agent: data[:user_agent],referrer: data[:tracking_referrer],session_id: data[:tracking_session_id],diff --git a/spec/jobs/detect_crawler_pageviews_spec.rb b/spec/jobs/detect_crawler_pageviews_spec.rbnew file mode 100644index 00000000000..defc915fc8f--- /dev/null+++ b/spec/jobs/detect_crawler_pageviews_spec.rb@@ -0,0 +1,19 @@+# frozen_string_literal: true++RSpec.describe Jobs::DetectCrawlerPageviews do+ it "does nothing when detection is disabled" do+ SiteSetting.experimental_detect_crawler_pageviews = false+ CrawlerScorer.expects(:score!).never++ described_class.new.execute({})+ end++ it "scores the last hour of pageviews when enabled" do+ SiteSetting.experimental_detect_crawler_pageviews = true+ freeze_time++ CrawlerScorer.expects(:score!).with(window_start: 1.hour.ago, window_end: Time.now)++ described_class.new.execute({})+ end+enddiff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rbnew file mode 100644index 00000000000..04a1cb4cab4--- /dev/null+++ b/spec/lib/crawler_scorer_spec.rb@@ -0,0 +1,122 @@+# frozen_string_literal: true++RSpec.describe CrawlerScorer do+ let(:hostname) { Discourse.current_hostname }++ def make_event(opts = {})+ defaults = {+ url: "/t/topic/1",+ ip_address: "1.2.3.4",+ user_agent: "Mozilla/5.0",+ session_id: SecureRandom.hex(8),+ created_at: 30.minutes.ago,+ referrer: "https://#{hostname}/",+ }+ BrowserPageviewEvent.create!(defaults.merge(opts))+ end++ def score!+ described_class.score!(window_start: 1.hour.ago, window_end: Time.now)+ end++ it "scores automation user agents at +100" do+ event = make_event(user_agent: "Mozilla/5.0 (X11; Linux x86_64) HeadlessChrome/120.0.0.0")+ score!+ expect(event.reload.score).to eq(100)+ end++ it "scores known crawler ASNs at +15" do+ SiteSetting.crawler_asns = "12345"+ event = make_event(asn: 12_345)+ score!+ expect(event.reload.score).to eq(15)+ end++ it "scores pageview velocity at or above VELOCITY_LOW threshold at +10" do+ stub_const(CrawlerScorer, :VELOCITY_LOW, 10) do+ session_id = "burst-session"+ base = 30.minutes.ago+ 10.times { |i| make_event(session_id: session_id, created_at: base + (i * 15).seconds) }++ score!++ expect(+ BrowserPageviewEvent.where(session_id: session_id).pluck(:score).uniq,+ ).to contain_exactly(10)+ end+ end++ it "scores session churn when one ip+ua spawns many short sessions" do+ stub_const(CrawlerScorer, :CHURN_HIGH_MIN_SESSIONS, 3) do+ 3.times do |i|+ make_event(ip_address: "9.9.9.9", user_agent: "ScriptyBot/1.0", session_id: "churn-#{i}")+ end++ score!++ expect(+ BrowserPageviewEvent.where(ip_address: "9.9.9.9").pluck(:score).uniq,+ ).to contain_exactly(20)+ end+ end++ it "scores rapid navigation when the median gap is under 5 seconds" do+ stub_const(CrawlerScorer, :RAPID_NAV_MIN_GAPS, 3) do+ session_id = "rapid-session"+ base = 30.minutes.ago+ 4.times { |i| make_event(session_id: session_id, created_at: base + i.seconds) }++ score!++ expect(+ BrowserPageviewEvent.where(session_id: session_id).pluck(:score).uniq,+ ).to contain_exactly(15)+ end+ end++ it "scores referrer discontinuity when most pageviews have no referrer" do+ stub_const(CrawlerScorer, :REFERRER_MIN_EVENTS, 2) do+ session_id = "ref-session"+ 2.times do+ make_event(+ ip_address: "5.5.5.5",+ user_agent: "RefBot/1.0",+ session_id: session_id,+ referrer: nil,+ )+ end++ score!++ expect(+ BrowserPageviewEvent.where(ip_address: "5.5.5.5").pluck(:score).uniq,+ ).to contain_exactly(10)+ end+ end++ it "also scores logged-in events" do+ user = Fabricate(:user)+ event = make_event(user_id: user.id, user_agent: "Mozilla/5.0 HeadlessChrome/120.0.0.0")++ score!++ expect(event.reload.score).to eq(100)+ end++ it "ignores events outside the window" do+ event = make_event(user_agent: "HeadlessChrome/120", created_at: 2.hours.ago)++ score!++ expect(event.reload.score).to be_nil+ end++ it "only updates the score when the new value is higher" do+ event = make_event(user_agent: "Mozilla/5.0 HeadlessChrome/120")+ event.update!(score: 120)++ score!++ expect(event.reload.score).to eq(120)+ end+end