Skip to content

A platform for community discussion. Free, open, simple.

DEV: Score sessions that rotate IP addresses in `CrawlerScorer` (#42083)

Krzysztof Kotlarek03a7c723a2a4
diff --git a/app/models/browser_pageview_event_score.rb b/app/models/browser_pageview_event_score.rb
index d65892ec15b..8f0462677a5 100644
--- a/app/models/browser_pageview_event_score.rb
+++ b/app/models/browser_pageview_event_score.rb
@@ -12,6 +12,7 @@ end
# automation_ua_score :integer default(0), not null
# churn_score :integer default(0), not null
# engagement_score :integer default(0), not null
+# ip_rotation_score :integer default(0), not null
# known_asn_score :integer default(0), not null
# rapid_nav_score :integer default(0), not null
# referrer_score :integer default(0), not null
diff --git a/config/site_settings.yml b/config/site_settings.yml
index a52e8d53c6e..0440fcf6f77 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -3121,9 +3121,10 @@ security:
# 16276=OVH, 24940=Hetzner, 136907=Huawei Cloud, 51747=Internet Vikings,
# 4134/4837=China Telecom/Unicom, 5089=Virgin, 55836=Reliance Jio,
# 5410=Bouygues, 701=Verizon, 6805=Telefonica, 45899=VNPT
+ # 55967=Baidu
crawler_asns:
hidden: true
- default: "16509|16276|714|13238|14618|136907|32934|15169|4134|4837|24940|396982|8075|51747|140577|45899|210743|13335|36040|5089|55836|5410|701|7941|6805"
+ default: "16509|16276|714|13238|14618|136907|32934|15169|4134|4837|24940|396982|8075|51747|140577|45899|210743|13335|36040|5089|55836|5410|701|7941|6805|55967"
type: list
list_type: compact
cors_origins:
diff --git a/db/migrate/20260728071552_add_ip_rotation_score_to_browser_pageview_event_scores.rb b/db/migrate/20260728071552_add_ip_rotation_score_to_browser_pageview_event_scores.rb
new file mode 100644
index 00000000000..517a8091ad5
--- /dev/null
+++ b/db/migrate/20260728071552_add_ip_rotation_score_to_browser_pageview_event_scores.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+class AddIpRotationScoreToBrowserPageviewEventScores < ActiveRecord::Migration[8.0]
+ def change
+ add_column :browser_pageview_event_scores,
+ :ip_rotation_score,
+ :smallint,
+ null: false,
+ default: 0
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index c927f186b0f..42850b77fdb 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1991,7 +1991,8 @@ CREATE TABLE public.browser_pageview_event_scores (
churn_score smallint DEFAULT 0 NOT NULL,
rapid_nav_score smallint DEFAULT 0 NOT NULL,
referrer_score smallint DEFAULT 0 NOT NULL,
- engagement_score smallint DEFAULT 0 NOT NULL
+ engagement_score smallint DEFAULT 0 NOT NULL,
+ ip_rotation_score smallint DEFAULT 0 NOT NULL
);
@@ -23050,6 +23051,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groups
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
+('20260728071552'),
('20260727035337'),
('20260723100008'),
('20260723013754'),
diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rb
index 7f5e4d69315..ac557120c46 100644
--- a/lib/crawler_scorer.rb
+++ b/lib/crawler_scorer.rb
@@ -24,6 +24,10 @@ class CrawlerScorer
RAPID_NAV_MAX_MEDIAN_SECONDS = 5
RAPID_NAV_SCORE = 15
+ IP_ROTATION_MIN_IPS = 3
+ IP_ROTATION_MAX_SECONDS_PER_CHANGE = 300
+ IP_ROTATION_SCORE = 30
+
REFERRER_MIN_EVENTS = 5
REFERRER_LOW_RATIO = 0.5
REFERRER_HIGH_RATIO = 0.8
@@ -59,6 +63,9 @@ class CrawlerScorer
rapid_nav_min_gaps: RAPID_NAV_MIN_GAPS,
rapid_nav_max_median_seconds: RAPID_NAV_MAX_MEDIAN_SECONDS,
rapid_nav_score: RAPID_NAV_SCORE,
+ ip_rotation_min_ips: IP_ROTATION_MIN_IPS,
+ ip_rotation_max_seconds_per_change: IP_ROTATION_MAX_SECONDS_PER_CHANGE,
+ ip_rotation_score: IP_ROTATION_SCORE,
referrer_min_events: REFERRER_MIN_EVENTS,
referrer_low_ratio: REFERRER_LOW_RATIO,
referrer_high_ratio: REFERRER_HIGH_RATIO,
@@ -122,6 +129,16 @@ class CrawlerScorer
GROUP BY ip_address, user_agent, source
),
+ session_stats AS (
+ SELECT
+ session_id,
+ source,
+ COUNT(DISTINCT ip_address) AS distinct_ips,
+ EXTRACT(EPOCH FROM MAX(created_at) - MIN(created_at)) AS span_seconds
+ FROM events
+ GROUP BY session_id, source
+ ),
+
breakdown AS (
SELECT
e.id,
@@ -154,6 +171,13 @@ class CrawlerScorer
THEN :rapid_nav_score
ELSE 0
END AS rapid_nav_score,
+ CASE
+ WHEN ss.distinct_ips >= :ip_rotation_min_ips
+ AND (ss.distinct_ips - 1) * :ip_rotation_max_seconds_per_change
+ > GREATEST(ss.span_seconds, 1)
+ THEN :ip_rotation_score
+ ELSE 0
+ END AS ip_rotation_score,
CASE
WHEN iu.pageviews >= :referrer_min_events
AND iu.bad_referrer_ratio >= :referrer_high_ratio THEN :referrer_high_score
@@ -168,6 +192,7 @@ class CrawlerScorer
FROM events e
LEFT JOIN ipua_stats iu USING (ip_address, user_agent, source)
LEFT JOIN median_gap mg USING (ip_address, user_agent, source)
+ LEFT JOIN session_stats ss USING (session_id, source)
LEFT JOIN browser_pageview_session_engagements se
ON se.session_id = e.session_id
AND (
@@ -183,16 +208,17 @@ class CrawlerScorer
velocity_score,
churn_score,
rapid_nav_score,
+ ip_rotation_score,
referrer_score,
engagement_score,
GREATEST(
0,
automation_ua_score + known_asn_score + velocity_score + churn_score
- + rapid_nav_score + referrer_score + engagement_score
+ + rapid_nav_score + ip_rotation_score + referrer_score + engagement_score
) AS score
FROM breakdown
WHERE automation_ua_score + known_asn_score + velocity_score + churn_score
- + rapid_nav_score + referrer_score > 0
+ + rapid_nav_score + ip_rotation_score + referrer_score > 0
),
updated AS (
@@ -207,6 +233,7 @@ class CrawlerScorer
t.velocity_score,
t.churn_score,
t.rapid_nav_score,
+ t.ip_rotation_score,
t.referrer_score,
t.engagement_score
)
@@ -218,6 +245,7 @@ class CrawlerScorer
velocity_score,
churn_score,
rapid_nav_score,
+ ip_rotation_score,
referrer_score,
engagement_score
)
@@ -228,6 +256,7 @@ class CrawlerScorer
velocity_score,
churn_score,
rapid_nav_score,
+ ip_rotation_score,
referrer_score,
engagement_score
FROM updated
@@ -237,6 +266,7 @@ class CrawlerScorer
velocity_score = EXCLUDED.velocity_score,
churn_score = EXCLUDED.churn_score,
rapid_nav_score = EXCLUDED.rapid_nav_score,
+ ip_rotation_score = EXCLUDED.ip_rotation_score,
referrer_score = EXCLUDED.referrer_score,
engagement_score = EXCLUDED.engagement_score;
SQL
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb
index 126c92cdbb5..9cc5ed16ab2 100644
--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb
+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb
@@ -1545,6 +1545,7 @@ module DiscourseDataExplorer
MAX(s.velocity_score) AS velocity,
MAX(s.churn_score) AS churn,
MAX(s.rapid_nav_score) AS rapid_nav,
+ MAX(s.ip_rotation_score) AS ip_rotation,
MAX(s.referrer_score) AS referrer,
NULLIF(
CONCAT_WS(', ',
@@ -1553,6 +1554,7 @@ module DiscourseDataExplorer
CASE WHEN MAX(s.velocity_score) > 0 THEN 'high velocity (+' || MAX(s.velocity_score) || ')' END,
CASE WHEN MAX(s.churn_score) > 0 THEN 'session churn (+' || MAX(s.churn_score) || ')' END,
CASE WHEN MAX(s.rapid_nav_score) > 0 THEN 'rapid navigation' END,
+ CASE WHEN MAX(s.ip_rotation_score) > 0 THEN 'ip rotation (+' || MAX(s.ip_rotation_score) || ')' END,
CASE WHEN MAX(s.referrer_score) > 0 THEN 'bad referrer (+' || MAX(s.referrer_score) || ')' END
),
''
diff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rb
index 15c88b1e013..08ae1fffaa3 100644
--- a/spec/lib/crawler_scorer_spec.rb
+++ b/spec/lib/crawler_scorer_spec.rb
@@ -42,6 +42,7 @@ RSpec.describe CrawlerScorer do
expect(breakdown.velocity_score).to eq(0)
expect(breakdown.churn_score).to eq(0)
expect(breakdown.rapid_nav_score).to eq(0)
+ expect(breakdown.ip_rotation_score).to eq(0)
expect(breakdown.referrer_score).to eq(0)
end
@@ -101,6 +102,65 @@ RSpec.describe CrawlerScorer do
end
end
+ it "scores ip rotation when one session is replayed through several addresses" do
+ base = 30.minutes.ago
+ %w[202.46.62.21 202.46.62.55 202.46.62.97].each_with_index do |ip, i|
+ make_event(ip_address: ip, session_id: "rotating-session", created_at: base + (i * 5).seconds)
+ end
+
+ score!
+
+ expect(
+ BrowserPageviewEvent.where(session_id: "rotating-session").pluck(:score).uniq,
+ ).to contain_exactly(30)
+ end
+
+ it "scores ip rotation the same whether or not the addresses cross ASNs" do
+ base = 30.minutes.ago
+ [
+ ["202.46.62.21", 64_496],
+ ["202.46.62.55", 64_496],
+ ["1.2.3.9", 64_497],
+ ].each_with_index do |(ip, asn), i|
+ make_event(
+ ip_address: ip,
+ asn: asn,
+ session_id: "cross-asn-session",
+ created_at: base + (i * 5).seconds,
+ )
+ end
+
+ score!
+
+ expect(
+ BrowserPageviewEvent.where(session_id: "cross-asn-session").pluck(:score).uniq,
+ ).to contain_exactly(30)
+ end
+
+ it "does not score a session that only ever uses two addresses" do
+ base = 30.minutes.ago
+ %w[10.0.0.1 10.0.0.2].each_with_index do |ip, i|
+ make_event(ip_address: ip, session_id: "handover-session", created_at: base + (i * 5).seconds)
+ end
+
+ score!
+
+ expect(BrowserPageviewEvent.where(session_id: "handover-session").pluck(:score).uniq).to eq(
+ [nil],
+ )
+ end
+
+ it "does not score ip changes slow enough to be a network handover" do
+ base = 40.minutes.ago
+ %w[10.0.0.1 10.0.0.2 10.0.0.3].each_with_index do |ip, i|
+ make_event(ip_address: ip, session_id: "slow-session", created_at: base + (i * 11).minutes)
+ end
+
+ score!
+
+ expect(BrowserPageviewEvent.where(session_id: "slow-session").pluck(:score).uniq).to eq([nil])
+ end
+
it "scores referrer discontinuity when most pageviews have no referrer" do
stub_const(CrawlerScorer, :REFERRER_MIN_EVENTS, 2) do
session_id = "ref-session"