A platform for community discussion. Free, open, simple.
DEV: Score sessions that rotate IP addresses in `CrawlerScorer` (#42083)
Krzysztof Kotlarek03a7c723a2a4
diff --git a/app/models/browser_pageview_event_score.rb b/app/models/browser_pageview_event_score.rbindex d65892ec15b..8f0462677a5 100644--- a/app/models/browser_pageview_event_score.rb+++ b/app/models/browser_pageview_event_score.rb@@ -12,6 +12,7 @@ end# automation_ua_score :integer default(0), not null# churn_score :integer default(0), not null# engagement_score :integer default(0), not null+# ip_rotation_score :integer default(0), not null# known_asn_score :integer default(0), not null# rapid_nav_score :integer default(0), not null# referrer_score :integer default(0), not nulldiff --git a/config/site_settings.yml b/config/site_settings.ymlindex a52e8d53c6e..0440fcf6f77 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -3121,9 +3121,10 @@ security:# 16276=OVH, 24940=Hetzner, 136907=Huawei Cloud, 51747=Internet Vikings,# 4134/4837=China Telecom/Unicom, 5089=Virgin, 55836=Reliance Jio,# 5410=Bouygues, 701=Verizon, 6805=Telefonica, 45899=VNPT+ # 55967=Baiducrawler_asns:hidden: true- default: "16509|16276|714|13238|14618|136907|32934|15169|4134|4837|24940|396982|8075|51747|140577|45899|210743|13335|36040|5089|55836|5410|701|7941|6805"+ default: "16509|16276|714|13238|14618|136907|32934|15169|4134|4837|24940|396982|8075|51747|140577|45899|210743|13335|36040|5089|55836|5410|701|7941|6805|55967"type: listlist_type: compactcors_origins:diff --git a/db/migrate/20260728071552_add_ip_rotation_score_to_browser_pageview_event_scores.rb b/db/migrate/20260728071552_add_ip_rotation_score_to_browser_pageview_event_scores.rbnew file mode 100644index 00000000000..517a8091ad5--- /dev/null+++ b/db/migrate/20260728071552_add_ip_rotation_score_to_browser_pageview_event_scores.rb@@ -0,0 +1,10 @@+# frozen_string_literal: true+class AddIpRotationScoreToBrowserPageviewEventScores < ActiveRecord::Migration[8.0]+ def change+ add_column :browser_pageview_event_scores,+ :ip_rotation_score,+ :smallint,+ null: false,+ default: 0+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex c927f186b0f..42850b77fdb 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -1991,7 +1991,8 @@ CREATE TABLE public.browser_pageview_event_scores (churn_score smallint DEFAULT 0 NOT NULL,rapid_nav_score smallint DEFAULT 0 NOT NULL,referrer_score smallint DEFAULT 0 NOT NULL,- engagement_score smallint DEFAULT 0 NOT NULL+ engagement_score smallint DEFAULT 0 NOT NULL,+ ip_rotation_score smallint DEFAULT 0 NOT NULL);@@ -23050,6 +23051,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groupsSET search_path TO "$user", public;INSERT INTO "schema_migrations" (version) VALUES+('20260728071552'),('20260727035337'),('20260723100008'),('20260723013754'),diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rbindex 7f5e4d69315..ac557120c46 100644--- a/lib/crawler_scorer.rb+++ b/lib/crawler_scorer.rb@@ -24,6 +24,10 @@ class CrawlerScorerRAPID_NAV_MAX_MEDIAN_SECONDS = 5RAPID_NAV_SCORE = 15+ IP_ROTATION_MIN_IPS = 3+ IP_ROTATION_MAX_SECONDS_PER_CHANGE = 300+ IP_ROTATION_SCORE = 30+REFERRER_MIN_EVENTS = 5REFERRER_LOW_RATIO = 0.5REFERRER_HIGH_RATIO = 0.8@@ -59,6 +63,9 @@ class CrawlerScorerrapid_nav_min_gaps: RAPID_NAV_MIN_GAPS,rapid_nav_max_median_seconds: RAPID_NAV_MAX_MEDIAN_SECONDS,rapid_nav_score: RAPID_NAV_SCORE,+ ip_rotation_min_ips: IP_ROTATION_MIN_IPS,+ ip_rotation_max_seconds_per_change: IP_ROTATION_MAX_SECONDS_PER_CHANGE,+ ip_rotation_score: IP_ROTATION_SCORE,referrer_min_events: REFERRER_MIN_EVENTS,referrer_low_ratio: REFERRER_LOW_RATIO,referrer_high_ratio: REFERRER_HIGH_RATIO,@@ -122,6 +129,16 @@ class CrawlerScorerGROUP BY ip_address, user_agent, source),+ session_stats AS (+ SELECT+ session_id,+ source,+ COUNT(DISTINCT ip_address) AS distinct_ips,+ EXTRACT(EPOCH FROM MAX(created_at) - MIN(created_at)) AS span_seconds+ FROM events+ GROUP BY session_id, source+ ),+breakdown AS (SELECTe.id,@@ -154,6 +171,13 @@ class CrawlerScorerTHEN :rapid_nav_scoreELSE 0END AS rapid_nav_score,+ CASE+ WHEN ss.distinct_ips >= :ip_rotation_min_ips+ AND (ss.distinct_ips - 1) * :ip_rotation_max_seconds_per_change+ > GREATEST(ss.span_seconds, 1)+ THEN :ip_rotation_score+ ELSE 0+ END AS ip_rotation_score,CASEWHEN iu.pageviews >= :referrer_min_eventsAND iu.bad_referrer_ratio >= :referrer_high_ratio THEN :referrer_high_score@@ -168,6 +192,7 @@ class CrawlerScorerFROM events eLEFT JOIN ipua_stats iu USING (ip_address, user_agent, source)LEFT JOIN median_gap mg USING (ip_address, user_agent, source)+ LEFT JOIN session_stats ss USING (session_id, source)LEFT JOIN browser_pageview_session_engagements seON se.session_id = e.session_idAND (@@ -183,16 +208,17 @@ class CrawlerScorervelocity_score,churn_score,rapid_nav_score,+ ip_rotation_score,referrer_score,engagement_score,GREATEST(0,automation_ua_score + known_asn_score + velocity_score + churn_score- + rapid_nav_score + referrer_score + engagement_score+ + rapid_nav_score + ip_rotation_score + referrer_score + engagement_score) AS scoreFROM breakdownWHERE automation_ua_score + known_asn_score + velocity_score + churn_score- + rapid_nav_score + referrer_score > 0+ + rapid_nav_score + ip_rotation_score + referrer_score > 0),updated AS (@@ -207,6 +233,7 @@ class CrawlerScorert.velocity_score,t.churn_score,t.rapid_nav_score,+ t.ip_rotation_score,t.referrer_score,t.engagement_score)@@ -218,6 +245,7 @@ class CrawlerScorervelocity_score,churn_score,rapid_nav_score,+ ip_rotation_score,referrer_score,engagement_score)@@ -228,6 +256,7 @@ class CrawlerScorervelocity_score,churn_score,rapid_nav_score,+ ip_rotation_score,referrer_score,engagement_scoreFROM updated@@ -237,6 +266,7 @@ class CrawlerScorervelocity_score = EXCLUDED.velocity_score,churn_score = EXCLUDED.churn_score,rapid_nav_score = EXCLUDED.rapid_nav_score,+ ip_rotation_score = EXCLUDED.ip_rotation_score,referrer_score = EXCLUDED.referrer_score,engagement_score = EXCLUDED.engagement_score;SQLdiff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rbindex 126c92cdbb5..9cc5ed16ab2 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb@@ -1545,6 +1545,7 @@ module DiscourseDataExplorerMAX(s.velocity_score) AS velocity,MAX(s.churn_score) AS churn,MAX(s.rapid_nav_score) AS rapid_nav,+ MAX(s.ip_rotation_score) AS ip_rotation,MAX(s.referrer_score) AS referrer,NULLIF(CONCAT_WS(', ',@@ -1553,6 +1554,7 @@ module DiscourseDataExplorerCASE WHEN MAX(s.velocity_score) > 0 THEN 'high velocity (+' || MAX(s.velocity_score) || ')' END,CASE WHEN MAX(s.churn_score) > 0 THEN 'session churn (+' || MAX(s.churn_score) || ')' END,CASE WHEN MAX(s.rapid_nav_score) > 0 THEN 'rapid navigation' END,+ CASE WHEN MAX(s.ip_rotation_score) > 0 THEN 'ip rotation (+' || MAX(s.ip_rotation_score) || ')' END,CASE WHEN MAX(s.referrer_score) > 0 THEN 'bad referrer (+' || MAX(s.referrer_score) || ')' END),''diff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rbindex 15c88b1e013..08ae1fffaa3 100644--- a/spec/lib/crawler_scorer_spec.rb+++ b/spec/lib/crawler_scorer_spec.rb@@ -42,6 +42,7 @@ RSpec.describe CrawlerScorer doexpect(breakdown.velocity_score).to eq(0)expect(breakdown.churn_score).to eq(0)expect(breakdown.rapid_nav_score).to eq(0)+ expect(breakdown.ip_rotation_score).to eq(0)expect(breakdown.referrer_score).to eq(0)end@@ -101,6 +102,65 @@ RSpec.describe CrawlerScorer doendend+ it "scores ip rotation when one session is replayed through several addresses" do+ base = 30.minutes.ago+ %w[202.46.62.21 202.46.62.55 202.46.62.97].each_with_index do |ip, i|+ make_event(ip_address: ip, session_id: "rotating-session", created_at: base + (i * 5).seconds)+ end++ score!++ expect(+ BrowserPageviewEvent.where(session_id: "rotating-session").pluck(:score).uniq,+ ).to contain_exactly(30)+ end++ it "scores ip rotation the same whether or not the addresses cross ASNs" do+ base = 30.minutes.ago+ [+ ["202.46.62.21", 64_496],+ ["202.46.62.55", 64_496],+ ["1.2.3.9", 64_497],+ ].each_with_index do |(ip, asn), i|+ make_event(+ ip_address: ip,+ asn: asn,+ session_id: "cross-asn-session",+ created_at: base + (i * 5).seconds,+ )+ end++ score!++ expect(+ BrowserPageviewEvent.where(session_id: "cross-asn-session").pluck(:score).uniq,+ ).to contain_exactly(30)+ end++ it "does not score a session that only ever uses two addresses" do+ base = 30.minutes.ago+ %w[10.0.0.1 10.0.0.2].each_with_index do |ip, i|+ make_event(ip_address: ip, session_id: "handover-session", created_at: base + (i * 5).seconds)+ end++ score!++ expect(BrowserPageviewEvent.where(session_id: "handover-session").pluck(:score).uniq).to eq(+ [nil],+ )+ end++ it "does not score ip changes slow enough to be a network handover" do+ base = 40.minutes.ago+ %w[10.0.0.1 10.0.0.2 10.0.0.3].each_with_index do |ip, i|+ make_event(ip_address: ip, session_id: "slow-session", created_at: base + (i * 11).minutes)+ end++ score!++ expect(BrowserPageviewEvent.where(session_id: "slow-session").pluck(:score).uniq).to eq([nil])+ end+it "scores referrer discontinuity when most pageviews have no referrer" dostub_const(CrawlerScorer, :REFERRER_MIN_EVENTS, 2) dosession_id = "ref-session"