A platform for community discussion. Free, open, simple.
FIX: Tighten crawler detection against low-effort human signals (#42498)
Krzysztof Kotlareka2e4474f0a9b
diff --git a/app/models/browser_pageview_session_engagement.rb b/app/models/browser_pageview_session_engagement.rbindex f2b618c5b4a..96e2a286dce 100644--- a/app/models/browser_pageview_session_engagement.rb+++ b/app/models/browser_pageview_session_engagement.rb@@ -15,6 +15,13 @@ class BrowserPageviewSessionEngagement < ActiveRecord::BaseGREATEST_COLUMNS = INTERACTION_COLUMNS + %i[engaged_seconds time_to_first_interaction_ms]+ MIN_DISTINCT_INTERACTIONS = 2++ def self.engaged_sql(table)+ signals = INTERACTION_COLUMNS.map { |column| "(#{table}.#{column} > 0)::int" }.join(" + ")+ "(#{signals}) >= #{MIN_DISTINCT_INTERACTIONS}"+ end+def self.upsert_from_payload(session_id:,mouse_move_events:,diff --git a/config/site_settings.yml b/config/site_settings.ymlindex f8c8dd8bc88..581f3564976 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -3127,10 +3127,10 @@ security:hidden: truedefault: "cubot|MetaIAB"# 38365/55967=Baidu, 140577=Ahrefs, 210743=Babbar, 7941=Internet Archive,- # 13238=Yandex+ # 13238=Yandex, 45102=Alibabacrawler_asns:hidden: true- default: "38365|55967|140577|210743|7941|13238"+ default: "38365|55967|140577|210743|7941|13238|45102"type: listlist_type: compact# 16509/14618=AWS, 16276=OVH, 136907/55990=Huawei Cloud,diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rbindex 2f6c2044246..2cc0dc0165f 100644--- a/lib/crawler_scorer.rb+++ b/lib/crawler_scorer.rb@@ -113,6 +113,7 @@ class CrawlerScorerFROM browser_pageview_events eLEFT JOIN browser_pageview_session_engagements seON se.session_id = e.session_id+ AND #{BrowserPageviewSessionEngagement.engaged_sql("se")}WHERE e.created_at >= :window_startAND e.created_at < :window_end),@@ -127,6 +128,7 @@ class CrawlerScorerFROM browser_pageview_events eLEFT JOIN browser_pageview_session_engagements seON se.session_id = e.session_id+ AND #{BrowserPageviewSessionEngagement.engaged_sql("se")}WHERE e.created_at >= :engagement_lookback_startAND e.created_at < :window_endAND EXISTS (diff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rbindex 266b7d9fd11..b7290246610 100644--- a/spec/lib/crawler_scorer_spec.rb+++ b/spec/lib/crawler_scorer_spec.rb@@ -127,7 +127,12 @@ RSpec.describe CrawlerScorer doit "does not score stale Chromium versions when the session has interaction" doSiteSetting.crawler_asns = "12345"event = make_event(asn: 12_345, user_agent: "Mozilla/5.0 Chrome/125.0.0.0 Safari/537.36")- Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, click_events: 1)+ Fabricate(+ :browser_pageview_session_engagement,+ session_id: event.session_id,+ click_events: 1,+ scroll_events: 3,+ )score!@@ -307,7 +312,12 @@ RSpec.describe CrawlerScorer doit "never scores an event below its bot signals" doSiteSetting.crawler_asns = "12345"event = make_event(asn: 12_345)- Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, scroll_events: 3)+ Fabricate(+ :browser_pageview_session_engagement,+ session_id: event.session_id,+ scroll_events: 3,+ mouse_move_events: 6,+ )score!@@ -318,18 +328,20 @@ RSpec.describe CrawlerScorer doexpect(breakdown.engagement_score).to eq(0)end- it "treats any engagement row as engagement, since the client only beacons on activity" do- event = make_event(user_agent: "Mozilla/5.0 HeadlessChrome/120.0.0.0")+ it "does not treat a single interaction signal as engagement" do+ SiteSetting.crawler_asns = "12345"+ event = make_event(asn: 12_345)Fabricate(:browser_pageview_session_engagement,session_id: event.session_id,+ scroll_events: 40,engaged_seconds: 30,)score!- expect(event.reload.score).to eq(100)- expect(event.browser_pageview_event_score.engagement_score).to eq(0)+ expect(event.reload.score).to eq(70)+ expect(event.browser_pageview_event_score.engagement_score).to eq(40)endit "lowers the score when an engagement beacon arrives after an earlier run" do@@ -339,7 +351,12 @@ RSpec.describe CrawlerScorer doscore!expect(event.reload.score).to eq(70)- Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, key_events: 4)+ Fabricate(+ :browser_pageview_session_engagement,+ session_id: event.session_id,+ key_events: 4,+ mouse_move_events: 2,+ )score!expect(event.reload.score).to eq(30)@@ -353,7 +370,12 @@ RSpec.describe CrawlerScorer do10.times do |i|make_event(asn: 12_345, session_id: engaged_session, created_at: base + i.minutes)end- Fabricate(:browser_pageview_session_engagement, session_id: engaged_session, scroll_events: 8)+ Fabricate(+ :browser_pageview_session_engagement,+ session_id: engaged_session,+ scroll_events: 8,+ mouse_move_events: 12,+ )abandoned_tab = make_event(asn: 12_345, session_id: "abandoned-tab", created_at: base)@@ -369,7 +391,12 @@ RSpec.describe CrawlerScorer do10.times do |i|make_event(asn: 12_345, session_id: engaged_session, created_at: 3.hours.ago + i.minutes)end- Fabricate(:browser_pageview_session_engagement, session_id: engaged_session, scroll_events: 8)+ Fabricate(+ :browser_pageview_session_engagement,+ session_id: engaged_session,+ scroll_events: 8,+ mouse_move_events: 12,+ )tail = 3.times.map { |i| make_event(asn: 12_345, created_at: 30.minutes.ago + i.minutes) }@@ -397,7 +424,12 @@ RSpec.describe CrawlerScorer do10.times do |i|make_event(asn: 12_345, session_id: engaged_session, created_at: 7.hours.ago + i.minutes)end- Fabricate(:browser_pageview_session_engagement, session_id: engaged_session, scroll_events: 8)+ Fabricate(+ :browser_pageview_session_engagement,+ session_id: engaged_session,+ scroll_events: 8,+ mouse_move_events: 12,+ )event = make_event(asn: 12_345)@@ -412,7 +444,12 @@ RSpec.describe CrawlerScorer dobase = 30.minutes.agoengaged_session = "partly-engaged"make_event(asn: 12_345, session_id: engaged_session, created_at: base)- Fabricate(:browser_pageview_session_engagement, session_id: engaged_session, click_events: 2)+ Fabricate(+ :browser_pageview_session_engagement,+ session_id: engaged_session,+ click_events: 2,+ mouse_move_events: 9,+ )3.times do |i|make_event(asn: 12_345, session_id: "unengaged-#{i}", created_at: base + (i + 1).minutes)