Skip to content

A platform for community discussion. Free, open, simple.

FEAT: Use session engagement in CrawlerScorer (#41693)

Krzysztof Kotlarek2cf226d90fcf
diff --git a/app/jobs/scheduled/detect_crawler_pageviews.rb b/app/jobs/scheduled/detect_crawler_pageviews.rb
index 1e27489714f..c736e0f78b2 100644
--- a/app/jobs/scheduled/detect_crawler_pageviews.rb
+++ b/app/jobs/scheduled/detect_crawler_pageviews.rb
@@ -5,12 +5,13 @@ module Jobs
every 10.minutes
LOOKBACK = 1.hour
+ WINDOW_DELAY = BrowserPageviewSessionEngagement::BEACON_SETTLE_PERIOD
def execute(args)
return if !SiteSetting.experimental_detect_crawler_pageviews
- now = Time.now
- CrawlerScorer.score!(window_start: now - LOOKBACK, window_end: now)
+ window_end = Time.now - WINDOW_DELAY
+ CrawlerScorer.score!(window_start: window_end - LOOKBACK, window_end: window_end)
end
end
end
diff --git a/app/models/browser_pageview_event_score.rb b/app/models/browser_pageview_event_score.rb
index 7bfcd926b06..d65892ec15b 100644
--- a/app/models/browser_pageview_event_score.rb
+++ b/app/models/browser_pageview_event_score.rb
@@ -11,6 +11,7 @@ end
# id :bigint not null, primary key
# automation_ua_score :integer default(0), not null
# churn_score :integer default(0), not null
+# engagement_score :integer default(0), not null
# known_asn_score :integer default(0), not null
# rapid_nav_score :integer default(0), not null
# referrer_score :integer default(0), not null
diff --git a/app/models/browser_pageview_session_engagement.rb b/app/models/browser_pageview_session_engagement.rb
index 487540c5d86..3f155230c21 100644
--- a/app/models/browser_pageview_session_engagement.rb
+++ b/app/models/browser_pageview_session_engagement.rb
@@ -2,18 +2,19 @@
class BrowserPageviewSessionEngagement < ActiveRecord::Base
MAX_SESSION_ID_LENGTH = 32
+ BEACON_SETTLE_PERIOD = 10.minutes
- GREATEST_COLUMNS = %i[
+ INTERACTION_COLUMNS = %i[
mouse_move_events
click_events
key_events
scroll_events
touch_events
back_forward_events
- engaged_seconds
- time_to_first_interaction_ms
]
+ GREATEST_COLUMNS = INTERACTION_COLUMNS + %i[engaged_seconds time_to_first_interaction_ms]
+
def self.upsert_from_payload(
session_id:,
mouse_move_events:,
diff --git a/app/models/browser_pageview_session_engagement_daily_rollup.rb b/app/models/browser_pageview_session_engagement_daily_rollup.rb
index def526eb5eb..2a4b33f4ecc 100644
--- a/app/models/browser_pageview_session_engagement_daily_rollup.rb
+++ b/app/models/browser_pageview_session_engagement_daily_rollup.rb
@@ -2,8 +2,7 @@
class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Base
BOUNCE_ENGAGED_SECONDS_THRESHOLD = 10
- MIN_SESSION_AGE = 10.minutes
- private_constant :BOUNCE_ENGAGED_SECONDS_THRESHOLD, :MIN_SESSION_AGE
+ private_constant :BOUNCE_ENGAGED_SECONDS_THRESHOLD
def self.aggregate(start_date:, end_date:)
start_date = start_date.to_date
@@ -63,7 +62,7 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Base
SQL
start_date:,
end_date:,
- session_started_before: MIN_SESSION_AGE.ago,
+ session_started_before: BrowserPageviewSessionEngagement::BEACON_SETTLE_PERIOD.ago,
bounce_threshold: BOUNCE_ENGAGED_SECONDS_THRESHOLD,
)
end
diff --git a/db/migrate/20260706050124_add_engagement_score_to_browser_pageview_event_scores.rb b/db/migrate/20260706050124_add_engagement_score_to_browser_pageview_event_scores.rb
new file mode 100644
index 00000000000..dd01db9eba0
--- /dev/null
+++ b/db/migrate/20260706050124_add_engagement_score_to_browser_pageview_event_scores.rb
@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+class AddEngagementScoreToBrowserPageviewEventScores < ActiveRecord::Migration[8.0]
+ def change
+ add_column :browser_pageview_event_scores, :engagement_score, :smallint, null: false, default: 0
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index 7551652d219..b4f7d7e6f32 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1990,7 +1990,8 @@ CREATE TABLE public.browser_pageview_event_scores (
velocity_score smallint DEFAULT 0 NOT NULL,
churn_score smallint DEFAULT 0 NOT NULL,
rapid_nav_score smallint DEFAULT 0 NOT NULL,
- referrer_score smallint DEFAULT 0 NOT NULL
+ referrer_score smallint DEFAULT 0 NOT NULL,
+ engagement_score smallint DEFAULT 0 NOT NULL
);
@@ -22906,6 +22907,7 @@ INSERT INTO "schema_migrations" (version) VALUES
('20260707184146'),
('20260707013407'),
('20260706151932'),
+('20260706050124'),
('20260703164430'),
('20260703163425'),
('20260702102111'),
diff --git a/frontend/discourse/app/lib/automation-detection.js b/frontend/discourse/app/lib/automation-detection.js
new file mode 100644
index 00000000000..6615e7dfe63
--- /dev/null
+++ b/frontend/discourse/app/lib/automation-detection.js
@@ -0,0 +1,16 @@
+import { isRailsTesting, isTesting } from "discourse/lib/environment";
+
+export function isAutomationDetected() {
+ // Our own test suites drive the browser with WebDriver but still need
+ // pageview tracking to fire.
+ if (isTesting() || isRailsTesting()) {
+ return false;
+ }
+
+ return !!(
+ window.navigator.webdriver ||
+ window.Cypress ||
+ window._phantom ||
+ window.__nightmare
+ );
+}
diff --git a/frontend/discourse/app/services/human-activity-tracker.js b/frontend/discourse/app/services/human-activity-tracker.js
index 8773bf6a26c..1ee90f28f0e 100644
--- a/frontend/discourse/app/services/human-activity-tracker.js
+++ b/frontend/discourse/app/services/human-activity-tracker.js
@@ -1,5 +1,6 @@
import { cancel } from "@ember/runloop";
import Service, { service } from "@ember/service";
+import { isAutomationDetected } from "discourse/lib/automation-detection";
import getURL from "discourse/lib/get-url";
import discourseLater from "discourse/lib/later";
import {
@@ -29,6 +30,8 @@ export default class HumanActivityTracker extends Service {
now = () => performance.now();
+ trustedEvent = (event) => event.isTrusted;
+
transport = (body) => {
navigator.sendBeacon?.(
getURL(ENGAGEMENT_PATH),
@@ -58,6 +61,10 @@ export default class HumanActivityTracker extends Service {
}
start() {
+ if (isAutomationDetected()) {
+ return;
+ }
+
this.#sessionId = document.querySelector(
"meta[name=discourse-track-view-session-id]"
)?.content;
@@ -130,11 +137,19 @@ export default class HumanActivityTracker extends Service {
}
#handleActivity(event) {
+ if (!this.trustedEvent(event)) {
+ return;
+ }
+
this.#counts[EVENT_COUNTERS[event.type]]++;
this.#markInteraction();
}
#handleMouseMove(event) {
+ if (!this.trustedEvent(event)) {
+ return;
+ }
+
const position = { x: event.clientX, y: event.clientY };
if (this.#lastPosition) {
diff --git a/frontend/discourse/tests/unit/services/human-activity-tracker-test.js b/frontend/discourse/tests/unit/services/human-activity-tracker-test.js
index b30877f5c01..36d02600bc6 100644
--- a/frontend/discourse/tests/unit/services/human-activity-tracker-test.js
+++ b/frontend/discourse/tests/unit/services/human-activity-tracker-test.js
@@ -47,6 +47,7 @@ module("Unit | Service | human-activity-tracker", function (hooks) {
this.tracker = this.owner.lookup("service:human-activity-tracker");
this.tracker.now = () => this.clock.ms;
+ this.tracker.trustedEvent = () => true;
this.tracker.transport = (body) => this.sent.push(body);
this.tracker.scheduleFlush = (callback) => {
this.flushTick = callback;
@@ -67,6 +68,24 @@ module("Unit | Service | human-activity-tracker", function (hooks) {
assert.strictEqual(this.sent.length, 0);
});
+ test("ignores untrusted synthetic events", async function (assert) {
+ this.tracker.trustedEvent = (event) => event.isTrusted;
+
+ await triggerEvent(document.body, "keydown");
+ await triggerEvent(document.body, "mousedown");
+ await triggerEvent(document.body, "mousemove", {
+ clientX: 10,
+ clientY: 10,
+ });
+ await triggerEvent(document.body, "mousemove", {
+ clientX: 20,
+ clientY: 20,
+ });
+ pagehide();
+
+ assert.strictEqual(this.sent.length, 0);
+ });
+
test("counts interaction events and reports them on flush", function (assert) {
window.dispatchEvent(new Event("keydown"));
window.dispatchEvent(new Event("mousedown"));
diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rb
index 396fc61d0b1..7f5e4d69315 100644
--- a/lib/crawler_scorer.rb
+++ b/lib/crawler_scorer.rb
@@ -1,16 +1,18 @@
# frozen_string_literal: true
class CrawlerScorer
+ BOT_SCORE_THRESHOLD = 60
+
AUTOMATION_UA_SCORE = 100
KNOWN_ASN_SCORE = 15
- VELOCITY_LOW = 120
+ VELOCITY_LOW = 150
VELOCITY_MEDIUM = 300
VELOCITY_HIGH = 600
- VELOCITY_LOW_SCORE = 10
+ VELOCITY_LOW_SCORE = 15
VELOCITY_MEDIUM_SCORE = 20
- VELOCITY_HIGH_SCORE = 35
+ VELOCITY_HIGH_SCORE = 50
CHURN_LOW_MIN_SESSIONS = 5
CHURN_HIGH_MIN_SESSIONS = 10
@@ -28,6 +30,8 @@ class CrawlerScorer
REFERRER_LOW_SCORE = 5
REFERRER_HIGH_SCORE = 10
+ HUMAN_ACTIVITY_SCORE = -40
+
def self.score!(window_start:, window_end:)
crawler_asns = SiteSetting.crawler_asns_map.map(&:to_i)
@@ -60,6 +64,7 @@ class CrawlerScorer
referrer_high_ratio: REFERRER_HIGH_RATIO,
referrer_low_score: REFERRER_LOW_SCORE,
referrer_high_score: REFERRER_HIGH_SCORE,
+ human_activity_score: HUMAN_ACTIVITY_SCORE,
)
end
end
@@ -155,10 +160,19 @@ class CrawlerScorer
WHEN iu.pageviews >= :referrer_min_events
AND iu.bad_referrer_ratio >= :referrer_low_ratio THEN :referrer_low_score
ELSE 0
- END AS referrer_score
+ END AS referrer_score,
+ CASE
+ WHEN se.session_id IS NOT NULL THEN :human_activity_score
+ ELSE 0
+ END AS engagement_score
FROM events e
LEFT JOIN ipua_stats iu USING (ip_address, user_agent, source)
LEFT JOIN median_gap mg USING (ip_address, user_agent, source)
+ LEFT JOIN browser_pageview_session_engagements se
+ ON se.session_id = e.session_id
+ AND (
+ #{BrowserPageviewSessionEngagement::INTERACTION_COLUMNS.map { |column| "se.#{column} > 0" }.join(" OR ")}
+ )
),
totals AS (
@@ -170,9 +184,15 @@ class CrawlerScorer
churn_score,
rapid_nav_score,
referrer_score,
- automation_ua_score + known_asn_score + velocity_score
- + churn_score + rapid_nav_score + referrer_score AS score
+ engagement_score,
+ GREATEST(
+ 0,
+ automation_ua_score + known_asn_score + velocity_score + churn_score
+ + rapid_nav_score + referrer_score + engagement_score
+ ) AS score
FROM breakdown
+ WHERE automation_ua_score + known_asn_score + velocity_score + churn_score
+ + rapid_nav_score + referrer_score > 0
),
updated AS (
@@ -180,15 +200,15 @@ class CrawlerScorer
SET score = t.score
FROM totals t
WHERE e.id = t.id
- AND t.score > 0
- AND t.score > COALESCE(e.score, 0)
+ AND (e.score IS NULL OR t.score > e.score)
RETURNING e.id,
t.automation_ua_score,
t.known_asn_score,
t.velocity_score,
t.churn_score,
t.rapid_nav_score,
- t.referrer_score
+ t.referrer_score,
+ t.engagement_score
)
INSERT INTO browser_pageview_event_scores (
@@ -198,7 +218,8 @@ class CrawlerScorer
velocity_score,
churn_score,
rapid_nav_score,
- referrer_score
+ referrer_score,
+ engagement_score
)
SELECT
id,
@@ -207,7 +228,8 @@ class CrawlerScorer
velocity_score,
churn_score,
rapid_nav_score,
- referrer_score
+ referrer_score,
+ engagement_score
FROM updated
ON CONFLICT (event_id) DO UPDATE
SET automation_ua_score = EXCLUDED.automation_ua_score,
@@ -215,6 +237,7 @@ class CrawlerScorer
velocity_score = EXCLUDED.velocity_score,
churn_score = EXCLUDED.churn_score,
rapid_nav_score = EXCLUDED.rapid_nav_score,
- referrer_score = EXCLUDED.referrer_score;
+ referrer_score = EXCLUDED.referrer_score,
+ engagement_score = EXCLUDED.engagement_score;
SQL
end
diff --git a/spec/jobs/detect_crawler_pageviews_spec.rb b/spec/jobs/detect_crawler_pageviews_spec.rb
index defc915fc8f..91820354926 100644
--- a/spec/jobs/detect_crawler_pageviews_spec.rb
+++ b/spec/jobs/detect_crawler_pageviews_spec.rb
@@ -8,11 +8,11 @@ RSpec.describe Jobs::DetectCrawlerPageviews do
described_class.new.execute({})
end
- it "scores the last hour of pageviews when enabled" do
+ it "scores a one hour window delayed to let engagement beacons arrive" do
SiteSetting.experimental_detect_crawler_pageviews = true
freeze_time
- CrawlerScorer.expects(:score!).with(window_start: 1.hour.ago, window_end: Time.now)
+ CrawlerScorer.expects(:score!).with(window_start: 70.minutes.ago, window_end: 10.minutes.ago)
described_class.new.execute({})
end
diff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rb
index 29bab203d1d..15c88b1e013 100644
--- a/spec/lib/crawler_scorer_spec.rb
+++ b/spec/lib/crawler_scorer_spec.rb
@@ -59,7 +59,7 @@ RSpec.describe CrawlerScorer do
expect(event.reload.score).to eq(15)
end
- it "scores pageview velocity at or above VELOCITY_LOW threshold at +10" do
+ it "scores pageview velocity at or above VELOCITY_LOW threshold at +15" do
stub_const(CrawlerScorer, :VELOCITY_LOW, 10) do
session_id = "burst-session"
base = 30.minutes.ago
@@ -69,7 +69,7 @@ RSpec.describe CrawlerScorer do
expect(
BrowserPageviewEvent.where(session_id: session_id).pluck(:score).uniq,
- ).to contain_exactly(10)
+ ).to contain_exactly(15)
end
end
@@ -147,13 +147,67 @@ RSpec.describe CrawlerScorer do
expect(event.reload.score).to eq(120)
end
+ it "discounts events whose session shows human interaction" do
+ event = make_event(user_agent: "Mozilla/5.0 HeadlessChrome/120.0.0.0")
+ Fabricate(
+ :browser_pageview_session_engagement,
+ session_id: event.session_id,
+ mouse_move_events: 5,
+ click_events: 2,
+ )
+
+ score!
+
+ expect(event.reload.score).to eq(60)
+ expect(event.browser_pageview_event_score.engagement_score).to eq(-40)
+ end
+
+ it "writes a zero score when the discount cancels all bot signals" do
+ SiteSetting.crawler_asns = "12345"
+ event = make_event(asn: 12_345)
+ Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, scroll_events: 3)
+
+ score!
+
+ expect(event.reload.score).to eq(0)
+ breakdown = event.browser_pageview_event_score
+ expect(breakdown.known_asn_score).to eq(15)
+ expect(breakdown.engagement_score).to eq(-40)
+ end
+
+ it "does not discount engagement rows crafted without interaction counts" do
+ event = make_event(user_agent: "Mozilla/5.0 HeadlessChrome/120.0.0.0")
+ Fabricate(
+ :browser_pageview_session_engagement,
+ session_id: event.session_id,
+ engaged_seconds: 30,
+ )
+
+ score!
+
+ expect(event.reload.score).to eq(100)
+ end
+
+ it "does not lower a previously assigned score when engagement arrives later" do
+ SiteSetting.crawler_asns = "12345"
+ event = make_event(asn: 12_345)
+
+ score!
+ expect(event.reload.score).to eq(15)
+
+ Fabricate(:browser_pageview_session_engagement, session_id: event.session_id, key_events: 4)
+ score!
+
+ expect(event.reload.score).to eq(15)
+ end
+
it "scores each source but partitions velocity so transports do not inflate each other" do
stub_const(CrawlerScorer, :VELOCITY_LOW, 10) do
stub_const(CrawlerScorer, :VELOCITY_MEDIUM, 20) do
base = 30.minutes.ago
# Same ip+ua, split across two transports with 12 pageviews each. On
- # its own each source sits in the LOW velocity tier (+10). Combined they
+ # its own each source sits in the LOW velocity tier (+15). Combined they
# would be 24 pageviews and reach the MEDIUM tier (+20), so equal
# per-source scores prove the heuristics stay partitioned by source.
{
@@ -172,13 +226,13 @@ RSpec.describe CrawlerScorer do
.where(source: BrowserPageviewEvent::SOURCE_PIGGYBACK)
.pluck(:score)
.uniq,
- ).to contain_exactly(10)
+ ).to contain_exactly(15)
expect(
BrowserPageviewEvent
.where(source: BrowserPageviewEvent::SOURCE_BEACON)
.pluck(:score)
.uniq,
- ).to contain_exactly(10)
+ ).to contain_exactly(15)
end
end
end