A platform for community discussion. Free, open, simple.
FEAT: Use session engagement in CrawlerScorer (#41693)
Krzysztof Kotlarek2cf226d90fcf
diff --git a/app/jobs/scheduled/detect_crawler_pageviews.rb b/app/jobs/scheduled/detect_crawler_pageviews.rbindex 1e27489714f..c736e0f78b2 100644--- a/app/jobs/scheduled/detect_crawler_pageviews.rb+++ b/app/jobs/scheduled/detect_crawler_pageviews.rb@@ -5,12 +5,13 @@ module Jobsevery 10.minutesLOOKBACK = 1.hour+ WINDOW_DELAY = BrowserPageviewSessionEngagement::BEACON_SETTLE_PERIODdef execute(args)return if !SiteSetting.experimental_detect_crawler_pageviews- now = Time.now- CrawlerScorer.score!(window_start: now - LOOKBACK, window_end: now)+ window_end = Time.now - WINDOW_DELAY+ CrawlerScorer.score!(window_start: window_end - LOOKBACK, window_end: window_end)endendenddiff --git a/app/models/browser_pageview_event_score.rb b/app/models/browser_pageview_event_score.rbindex 7bfcd926b06..d65892ec15b 100644--- a/app/models/browser_pageview_event_score.rb+++ b/app/models/browser_pageview_event_score.rb@@ -11,6 +11,7 @@ end# id :bigint not null, primary key# automation_ua_score :integer default(0), not null# churn_score :integer default(0), not null+# engagement_score :integer default(0), not null# known_asn_score :integer default(0), not null# rapid_nav_score :integer default(0), not null# referrer_score :integer default(0), not nulldiff --git a/app/models/browser_pageview_session_engagement.rb b/app/models/browser_pageview_session_engagement.rbindex 487540c5d86..3f155230c21 100644--- a/app/models/browser_pageview_session_engagement.rb+++ b/app/models/browser_pageview_session_engagement.rb@@ -2,18 +2,19 @@class BrowserPageviewSessionEngagement < ActiveRecord::BaseMAX_SESSION_ID_LENGTH = 32+ BEACON_SETTLE_PERIOD = 10.minutes- GREATEST_COLUMNS = %i[+ INTERACTION_COLUMNS = %i[mouse_move_eventsclick_eventskey_eventsscroll_eventstouch_eventsback_forward_events- engaged_seconds- time_to_first_interaction_ms]+ GREATEST_COLUMNS = INTERACTION_COLUMNS + %i[engaged_seconds time_to_first_interaction_ms]+def self.upsert_from_payload(session_id:,mouse_move_events:,diff --git a/app/models/browser_pageview_session_engagement_daily_rollup.rb b/app/models/browser_pageview_session_engagement_daily_rollup.rbindex def526eb5eb..2a4b33f4ecc 100644--- a/app/models/browser_pageview_session_engagement_daily_rollup.rb+++ b/app/models/browser_pageview_session_engagement_daily_rollup.rb@@ -2,8 +2,7 @@class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::BaseBOUNCE_ENGAGED_SECONDS_THRESHOLD = 10- MIN_SESSION_AGE = 10.minutes- private_constant :BOUNCE_ENGAGED_SECONDS_THRESHOLD, :MIN_SESSION_AGE+ private_constant :BOUNCE_ENGAGED_SECONDS_THRESHOLDdef self.aggregate(start_date:, end_date:)start_date = start_date.to_date@@ -63,7 +62,7 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::BaseSQLstart_date:,end_date:,- session_started_before: MIN_SESSION_AGE.ago,+ session_started_before: BrowserPageviewSessionEngagement::BEACON_SETTLE_PERIOD.ago,bounce_threshold: BOUNCE_ENGAGED_SECONDS_THRESHOLD,)enddiff --git a/db/migrate/20260706050124_add_engagement_score_to_browser_pageview_event_scores.rb b/db/migrate/20260706050124_add_engagement_score_to_browser_pageview_event_scores.rbnew file mode 100644index 00000000000..dd01db9eba0--- /dev/null+++ b/db/migrate/20260706050124_add_engagement_score_to_browser_pageview_event_scores.rb@@ -0,0 +1,6 @@+# frozen_string_literal: true+class AddEngagementScoreToBrowserPageviewEventScores < ActiveRecord::Migration[8.0]+ def change+ add_column :browser_pageview_event_scores, :engagement_score, :smallint, null: false, default: 0+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex 7551652d219..b4f7d7e6f32 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -1990,7 +1990,8 @@ CREATE TABLE public.browser_pageview_event_scores (velocity_score smallint DEFAULT 0 NOT NULL,churn_score smallint DEFAULT 0 NOT NULL,rapid_nav_score smallint DEFAULT 0 NOT NULL,- referrer_score smallint DEFAULT 0 NOT NULL+ referrer_score smallint DEFAULT 0 NOT NULL,+ engagement_score smallint DEFAULT 0 NOT NULL);@@ -22906,6 +22907,7 @@ INSERT INTO "schema_migrations" (version) VALUES('20260707184146'),('20260707013407'),('20260706151932'),+('20260706050124'),('20260703164430'),('20260703163425'),('20260702102111'),diff --git a/frontend/discourse/app/lib/automation-detection.js b/frontend/discourse/app/lib/automation-detection.jsnew file mode 100644index 00000000000..6615e7dfe63--- /dev/null+++ b/frontend/discourse/app/lib/automation-detection.js@@ -0,0 +1,16 @@+import { isRailsTesting, isTesting } from "discourse/lib/environment";++export function isAutomationDetected() {+ // Our own test suites drive the browser with WebDriver but still need+ // pageview tracking to fire.+ if (isTesting() || isRailsTesting()) {+ return false;+ }++ return !!(+ window.navigator.webdriver ||+ window.Cypress ||+ window._phantom ||+ window.__nightmare+ );+}diff --git a/frontend/discourse/app/services/human-activity-tracker.js b/frontend/discourse/app/services/human-activity-tracker.jsindex 8773bf6a26c..1ee90f28f0e 100644--- a/frontend/discourse/app/services/human-activity-tracker.js+++ b/frontend/discourse/app/services/human-activity-tracker.js@@ -1,5 +1,6 @@import { cancel } from "@ember/runloop";import Service, { service } from "@ember/service";+import { isAutomationDetected } from "discourse/lib/automation-detection";import getURL from "discourse/lib/get-url";import discourseLater from "discourse/lib/later";import {@@ -29,6 +30,8 @@ export default class HumanActivityTracker extends Service {now = () => performance.now();+ trustedEvent = (event) => event.isTrusted;+transport = (body) => {navigator.sendBeacon?.(getURL(ENGAGEMENT_PATH),@@ -58,6 +61,10 @@ export default class HumanActivityTracker extends Service {}start() {+ if (isAutomationDetected()) {+ return;+ }+this.#sessionId = document.querySelector("meta[name=discourse-track-view-session-id]")?.content;@@ -130,11 +137,19 @@ export default class HumanActivityTracker extends Service {}#handleActivity(event) {+ if (!this.trustedEvent(event)) {+ return;+ }+this.#counts[EVENT_COUNTERS[event.type]]++;this.#markInteraction();}#handleMouseMove(event) {+ if (!this.trustedEvent(event)) {+ return;+ }+const position = { x: event.clientX, y: event.clientY };if (this.#lastPosition) {diff --git a/frontend/discourse/tests/unit/services/human-activity-tracker-test.js b/frontend/discourse/tests/unit/services/human-activity-tracker-test.jsindex b30877f5c01..36d02600bc6 100644--- a/frontend/discourse/tests/unit/services/human-activity-tracker-test.js+++ b/frontend/discourse/tests/unit/services/human-activity-tracker-test.js@@ -47,6 +47,7 @@ module("Unit | Service | human-activity-tracker", function (hooks) {this.tracker = this.owner.lookup("service:human-activity-tracker");this.tracker.now = () => this.clock.ms;+ this.tracker.trustedEvent = () => true;this.tracker.transport = (body) => this.sent.push(body);this.tracker.scheduleFlush = (callback) => {this.flushTick = callback;@@ -67,6 +68,24 @@ module("Unit | Service | human-activity-tracker", function (hooks) {assert.strictEqual(this.sent.length, 0);});+ test("ignores untrusted synthetic events", async function (assert) {+ this.tracker.trustedEvent = (event) => event.isTrusted;++ await triggerEvent(document.body, "keydown");+ await triggerEvent(document.body, "mousedown");+ await triggerEvent(document.body, "mousemove", {+ clientX: 10,+ clientY: 10,+ });+ await triggerEvent(document.body, "mousemove", {+ clientX: 20,+ clientY: 20,+ });+ pagehide();++ assert.strictEqual(this.sent.length, 0);+ });+test("counts interaction events and reports them on flush", function (assert) {window.dispatchEvent(new Event("keydown"));window.dispatchEvent(new Event("mousedown"));diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rbindex 396fc61d0b1..7f5e4d69315 100644--- a/lib/crawler_scorer.rb+++ b/lib/crawler_scorer.rb@@ -1,16 +1,18 @@# frozen_string_literal: trueclass CrawlerScorer+ BOT_SCORE_THRESHOLD = 60+AUTOMATION_UA_SCORE = 100KNOWN_ASN_SCORE = 15- VELOCITY_LOW = 120+ VELOCITY_LOW = 150VELOCITY_MEDIUM = 300VELOCITY_HIGH = 600- VELOCITY_LOW_SCORE = 10+ VELOCITY_LOW_SCORE = 15VELOCITY_MEDIUM_SCORE = 20- VELOCITY_HIGH_SCORE = 35+ VELOCITY_HIGH_SCORE = 50CHURN_LOW_MIN_SESSIONS = 5CHURN_HIGH_MIN_SESSIONS = 10@@ -28,6 +30,8 @@ class CrawlerScorerREFERRER_LOW_SCORE = 5REFERRER_HIGH_SCORE = 10+ HUMAN_ACTIVITY_SCORE = -40+def self.score!(window_start:, window_end:)crawler_asns = SiteSetting.crawler_asns_map.map(&:to_i)@@ -60,6 +64,7 @@ class CrawlerScorerreferrer_high_ratio: REFERRER_HIGH_RATIO,referrer_low_score: REFERRER_LOW_SCORE,referrer_high_score: REFERRER_HIGH_SCORE,+ human_activity_score: HUMAN_ACTIVITY_SCORE,)endend@@ -155,10 +160,19 @@ class CrawlerScorerWHEN iu.pageviews >= :referrer_min_eventsAND iu.bad_referrer_ratio >= :referrer_low_ratio THEN :referrer_low_scoreELSE 0- END AS referrer_score+ END AS referrer_score,+ CASE+ WHEN se.session_id IS NOT NULL THEN :human_activity_score+ ELSE 0+ END AS engagement_scoreFROM events eLEFT JOIN ipua_stats iu USING (ip_address, user_agent, source)LEFT JOIN median_gap mg USING (ip_address, user_agent, source)+ LEFT JOIN browser_pageview_session_engagements se+ ON se.session_id = e.session_id+ AND (+ #{BrowserPageviewSessionEngagement::INTERACTION_COLUMNS.map { |column| "se.#{column} > 0" }.join(" OR ")}+ )),totals AS (@@ -170,9 +184,15 @@ class CrawlerScorerchurn_score,rapid_nav_score,referrer_score,- automation_ua_score + known_asn_score + velocity_score- + churn_score + rapid_nav_score + referrer_score AS score+ engagement_score,+ GREATEST(+ 0,+ automation_ua_score + known_asn_score + velocity_score + churn_score+ + rapid_nav_score + referrer_score + engagement_score+ ) AS scoreFROM breakdown+ WHERE automation_ua_score + known_asn_score + velocity_score + churn_score+ + rapid_nav_score + referrer_score > 0),updated AS (@@ -180,15 +200,15 @@ class CrawlerScorerSET score = t.scoreFROM totals tWHERE e.id = t.id- AND t.score > 0- AND t.score > COALESCE(e.score, 0)+ AND (e.score IS NULL OR t.score > e.score)RETURNING e.id,t.automation_ua_score,t.known_asn_score,t.velocity_score,t.churn_score,t.rapid_nav_score,- t.referrer_score+ t.referrer_score,+ t.engagement_score)INSERT INTO browser_pageview_event_scores (@@ -198,7 +218,8 @@ class CrawlerScorervelocity_score,churn_score,rapid_nav_score,- referrer_score+ referrer_score,+ engagement_score)SELECTid,@@ -207,7 +228,8 @@ class CrawlerScorervelocity_score,churn_score,rapid_nav_score,- referrer_score+ referrer_score,+ engagement_scoreFROM updatedON CONFLICT (event_id) DO UPDATESET automation_ua_score = EXCLUDED.automation_ua_score,@@ -215,6 +237,7 @@ class CrawlerScorervelocity_score = EXCLUDED.velocity_score,churn_score = EXCLUDED.churn_score,rapid_nav_score = EXCLUDED.rapid_nav_score,- referrer_score = EXCLUDED.referrer_score;+ referrer_score = EXCLUDED.referrer_score,+ engagement_score = EXCLUDED.engagement_score;SQLenddiff --git a/spec/jobs/detect_crawler_pageviews_spec.rb b/spec/jobs/detect_crawler_pageviews_spec.rbindex defc915fc8f..91820354926 100644--- a/spec/jobs/detect_crawler_pageviews_spec.rb+++ b/spec/jobs/detect_crawler_pageviews_spec.rb@@ -8,11 +8,11 @@ RSpec.describe Jobs::DetectCrawlerPageviews dodescribed_class.new.execute({})end- it "scores the last hour of pageviews when enabled" do+ it "scores a one hour window delayed to let engagement beacons arrive" doSiteSetting.experimental_detect_crawler_pageviews = truefreeze_time- CrawlerScorer.expects(:score!).with(window_start: 1.hour.ago, window_end: Time.now)+ CrawlerScorer.expects(:score!).with(window_start: 70.minutes.ago, window_end: 10.minutes.ago)described_class.new.execute({})enddiff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rbindex 29bab203d1d..15c88b1e013 100644--- a/spec/lib/crawler_scorer_spec.rb+++ b/spec/lib/crawler_scorer_spec.rb@@ -59,7 +59,7 @@ RSpec.describe CrawlerScorer doexpect(event.reload.score).to eq(15)end- it "scores pageview velocity at or above VELOCITY_LOW threshold at +10" do+ it "scores pageview velocity at or above VELOCITY_LOW threshold at +15" dostub_const(CrawlerScorer, :VELOCITY_LOW, 10) dosession_id = "burst-session"base = 30.minutes.ago@@ -69,7 +69,7 @@ RSpec.describe CrawlerScorer doexpect(BrowserPageviewEvent.where(session_id: session_id).pluck(:score).uniq,- ).to contain_exactly(10)+ ).to contain_exactly(15)endend@@ -147,13 +147,67 @@ RSpec.describe CrawlerScorer doexpect(event.reload.score).to eq(120)end+ it "discounts events whose session shows human interaction" do+ event = make_event(user_agent: "Mozilla/5.0 HeadlessChrome/120.0.0.0")+ Fabricate(+ :browser_pageview_session_engagement,+ session_id: event.session_id,+ mouse_move_events: 5,+ click_events: 2,+ )++ score!++ expect(event.reload.score).to eq(60)+ expect(event.browser_pageview_event_score.engagement_score).to eq(-40)+ end++ it "writes a zero score when the discount cancels all bot signals" do+ SiteSetting.crawler_asns = "12345"+ event = make_event(asn: 12_345)+ Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, scroll_events: 3)++ score!++ expect(event.reload.score).to eq(0)+ breakdown = event.browser_pageview_event_score+ expect(breakdown.known_asn_score).to eq(15)+ expect(breakdown.engagement_score).to eq(-40)+ end++ it "does not discount engagement rows crafted without interaction counts" do+ event = make_event(user_agent: "Mozilla/5.0 HeadlessChrome/120.0.0.0")+ Fabricate(+ :browser_pageview_session_engagement,+ session_id: event.session_id,+ engaged_seconds: 30,+ )++ score!++ expect(event.reload.score).to eq(100)+ end++ it "does not lower a previously assigned score when engagement arrives later" do+ SiteSetting.crawler_asns = "12345"+ event = make_event(asn: 12_345)++ score!+ expect(event.reload.score).to eq(15)++ Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, key_events: 4)+ score!++ expect(event.reload.score).to eq(15)+ end+it "scores each source but partitions velocity so transports do not inflate each other" dostub_const(CrawlerScorer, :VELOCITY_LOW, 10) dostub_const(CrawlerScorer, :VELOCITY_MEDIUM, 20) dobase = 30.minutes.ago# Same ip+ua, split across two transports with 12 pageviews each. On- # its own each source sits in the LOW velocity tier (+10). Combined they+ # its own each source sits in the LOW velocity tier (+15). Combined they# would be 24 pageviews and reach the MEDIUM tier (+20), so equal# per-source scores prove the heuristics stay partitioned by source.{@@ -172,13 +226,13 @@ RSpec.describe CrawlerScorer do.where(source: BrowserPageviewEvent::SOURCE_PIGGYBACK).pluck(:score).uniq,- ).to contain_exactly(10)+ ).to contain_exactly(15)expect(BrowserPageviewEvent.where(source: BrowserPageviewEvent::SOURCE_BEACON).pluck(:score).uniq,- ).to contain_exactly(10)+ ).to contain_exactly(15)endendend