Skip to content

A platform for community discussion. Free, open, simple.

FIX: Tighten crawler detection against low-effort human signals (#42498)

Krzysztof Kotlareka2e4474f0a9b
diff --git a/app/models/browser_pageview_session_engagement.rb b/app/models/browser_pageview_session_engagement.rb
index f2b618c5b4a..96e2a286dce 100644
--- a/app/models/browser_pageview_session_engagement.rb
+++ b/app/models/browser_pageview_session_engagement.rb
@@ -15,6 +15,13 @@ class BrowserPageviewSessionEngagement < ActiveRecord::Base
GREATEST_COLUMNS = INTERACTION_COLUMNS + %i[engaged_seconds time_to_first_interaction_ms]
+ MIN_DISTINCT_INTERACTIONS = 2
+
+ def self.engaged_sql(table)
+ signals = INTERACTION_COLUMNS.map { |column| "(#{table}.#{column} > 0)::int" }.join(" + ")
+ "(#{signals}) >= #{MIN_DISTINCT_INTERACTIONS}"
+ end
+
def self.upsert_from_payload(
session_id:,
mouse_move_events:,
diff --git a/config/site_settings.yml b/config/site_settings.yml
index f8c8dd8bc88..581f3564976 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -3127,10 +3127,10 @@ security:
hidden: true
default: "cubot|MetaIAB"
# 38365/55967=Baidu, 140577=Ahrefs, 210743=Babbar, 7941=Internet Archive,
- # 13238=Yandex
+ # 13238=Yandex, 45102=Alibaba
crawler_asns:
hidden: true
- default: "38365|55967|140577|210743|7941|13238"
+ default: "38365|55967|140577|210743|7941|13238|45102"
type: list
list_type: compact
# 16509/14618=AWS, 16276=OVH, 136907/55990=Huawei Cloud,
diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rb
index 2f6c2044246..2cc0dc0165f 100644
--- a/lib/crawler_scorer.rb
+++ b/lib/crawler_scorer.rb
@@ -113,6 +113,7 @@ class CrawlerScorer
FROM browser_pageview_events e
LEFT JOIN browser_pageview_session_engagements se
ON se.session_id = e.session_id
+ AND #{BrowserPageviewSessionEngagement.engaged_sql("se")}
WHERE e.created_at >= :window_start
AND e.created_at < :window_end
),
@@ -127,6 +128,7 @@ class CrawlerScorer
FROM browser_pageview_events e
LEFT JOIN browser_pageview_session_engagements se
ON se.session_id = e.session_id
+ AND #{BrowserPageviewSessionEngagement.engaged_sql("se")}
WHERE e.created_at >= :engagement_lookback_start
AND e.created_at < :window_end
AND EXISTS (
diff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rb
index 266b7d9fd11..b7290246610 100644
--- a/spec/lib/crawler_scorer_spec.rb
+++ b/spec/lib/crawler_scorer_spec.rb
@@ -127,7 +127,12 @@ RSpec.describe CrawlerScorer do
it "does not score stale Chromium versions when the session has interaction" do
SiteSetting.crawler_asns = "12345"
event = make_event(asn: 12_345, user_agent: "Mozilla/5.0 Chrome/125.0.0.0 Safari/537.36")
- Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, click_events: 1)
+ Fabricate(
+ :browser_pageview_session_engagement,
+ session_id: event.session_id,
+ click_events: 1,
+ scroll_events: 3,
+ )
score!
@@ -307,7 +312,12 @@ RSpec.describe CrawlerScorer do
it "never scores an event below its bot signals" do
SiteSetting.crawler_asns = "12345"
event = make_event(asn: 12_345)
- Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, scroll_events: 3)
+ Fabricate(
+ :browser_pageview_session_engagement,
+ session_id: event.session_id,
+ scroll_events: 3,
+ mouse_move_events: 6,
+ )
score!
@@ -318,18 +328,20 @@ RSpec.describe CrawlerScorer do
expect(breakdown.engagement_score).to eq(0)
end
- it "treats any engagement row as engagement, since the client only beacons on activity" do
- event = make_event(user_agent: "Mozilla/5.0 HeadlessChrome/120.0.0.0")
+ it "does not treat a single interaction signal as engagement" do
+ SiteSetting.crawler_asns = "12345"
+ event = make_event(asn: 12_345)
Fabricate(
:browser_pageview_session_engagement,
session_id: event.session_id,
+ scroll_events: 40,
engaged_seconds: 30,
)
score!
- expect(event.reload.score).to eq(100)
- expect(event.browser_pageview_event_score.engagement_score).to eq(0)
+ expect(event.reload.score).to eq(70)
+ expect(event.browser_pageview_event_score.engagement_score).to eq(40)
end
it "lowers the score when an engagement beacon arrives after an earlier run" do
@@ -339,7 +351,12 @@ RSpec.describe CrawlerScorer do
score!
expect(event.reload.score).to eq(70)
- Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, key_events: 4)
+ Fabricate(
+ :browser_pageview_session_engagement,
+ session_id: event.session_id,
+ key_events: 4,
+ mouse_move_events: 2,
+ )
score!
expect(event.reload.score).to eq(30)
@@ -353,7 +370,12 @@ RSpec.describe CrawlerScorer do
10.times do |i|
make_event(asn: 12_345, session_id: engaged_session, created_at: base + i.minutes)
end
- Fabricate(:browser_pageview_session_engagement, session_id: engaged_session, scroll_events: 8)
+ Fabricate(
+ :browser_pageview_session_engagement,
+ session_id: engaged_session,
+ scroll_events: 8,
+ mouse_move_events: 12,
+ )
abandoned_tab = make_event(asn: 12_345, session_id: "abandoned-tab", created_at: base)
@@ -369,7 +391,12 @@ RSpec.describe CrawlerScorer do
10.times do |i|
make_event(asn: 12_345, session_id: engaged_session, created_at: 3.hours.ago + i.minutes)
end
- Fabricate(:browser_pageview_session_engagement, session_id: engaged_session, scroll_events: 8)
+ Fabricate(
+ :browser_pageview_session_engagement,
+ session_id: engaged_session,
+ scroll_events: 8,
+ mouse_move_events: 12,
+ )
tail = 3.times.map { |i| make_event(asn: 12_345, created_at: 30.minutes.ago + i.minutes) }
@@ -397,7 +424,12 @@ RSpec.describe CrawlerScorer do
10.times do |i|
make_event(asn: 12_345, session_id: engaged_session, created_at: 7.hours.ago + i.minutes)
end
- Fabricate(:browser_pageview_session_engagement, session_id: engaged_session, scroll_events: 8)
+ Fabricate(
+ :browser_pageview_session_engagement,
+ session_id: engaged_session,
+ scroll_events: 8,
+ mouse_move_events: 12,
+ )
event = make_event(asn: 12_345)
@@ -412,7 +444,12 @@ RSpec.describe CrawlerScorer do
base = 30.minutes.ago
engaged_session = "partly-engaged"
make_event(asn: 12_345, session_id: engaged_session, created_at: base)
- Fabricate(:browser_pageview_session_engagement, session_id: engaged_session, click_events: 2)
+ Fabricate(
+ :browser_pageview_session_engagement,
+ session_id: engaged_session,
+ click_events: 2,
+ mouse_move_events: 9,
+ )
3.times do |i|
make_event(asn: 12_345, session_id: "unengaged-#{i}", created_at: base + (i + 1).minutes)