Skip to content

A platform for community discussion. Free, open, simple.

DEV: Track browser session engagement for crawler detection (#41215)

Krzysztof Kotlarekd570c043029a
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
index 7df092f4dd1..28009aae3f3 100644
--- a/app/helpers/application_helper.rb
+++ b/app/helpers/application_helper.rb
@@ -452,6 +452,11 @@ module ApplicationHelper
if UpcomingChanges.enabled?(:dashboard_improvements)
tags << tag.meta(name: "discourse-beacon-pageview-enabled", content: "true")
end
+
+ if UpcomingChanges.enabled?(:dashboard_improvements) &&
+ SiteSetting.persist_browser_pageview_events
+ tags << tag.meta(name: "discourse-engagement-tracking-enabled", content: "true")
+ end
tags.html_safe
end
diff --git a/app/jobs/scheduled/clean_up_browser_pageview_events.rb b/app/jobs/scheduled/clean_up_browser_pageview_events.rb
index 12ed20182f0..e3f741013ce 100644
--- a/app/jobs/scheduled/clean_up_browser_pageview_events.rb
+++ b/app/jobs/scheduled/clean_up_browser_pageview_events.rb
@@ -4,15 +4,23 @@ module Jobs
class CleanUpBrowserPageviewEvents < ::Jobs::Scheduled
every 1.day
+ BATCH_SIZE = 10_000
+
def execute(args)
return if !SiteSetting.clean_up_browser_pageview_events
+ cutoff = BrowserPageviewEvent.retention_cutoff
+
BrowserPageviewEvent
- .where("created_at < ?", BrowserPageviewEvent.retention_cutoff)
- .in_batches(of: 10_000) do |browser_pageview_events|
+ .where("created_at < ?", cutoff)
+ .in_batches(of: BATCH_SIZE) do |browser_pageview_events|
BrowserPageviewEventScore.where(event_id: browser_pageview_events.select(:id)).delete_all
browser_pageview_events.delete_all
end
+
+ BrowserPageviewSessionEngagement
+ .where("created_at < ?", cutoff)
+ .in_batches(of: BATCH_SIZE) { |session_engagements| session_engagements.delete_all }
end
end
end
diff --git a/app/models/browser_pageview_session_engagement.rb b/app/models/browser_pageview_session_engagement.rb
new file mode 100644
index 00000000000..487540c5d86
--- /dev/null
+++ b/app/models/browser_pageview_session_engagement.rb
@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+class BrowserPageviewSessionEngagement < ActiveRecord::Base
+ MAX_SESSION_ID_LENGTH = 32
+
+ GREATEST_COLUMNS = %i[
+ mouse_move_events
+ click_events
+ key_events
+ scroll_events
+ touch_events
+ back_forward_events
+ engaged_seconds
+ time_to_first_interaction_ms
+ ]
+
+ def self.upsert_from_payload(
+ session_id:,
+ mouse_move_events:,
+ click_events:,
+ key_events:,
+ scroll_events:,
+ touch_events:,
+ back_forward_events:,
+ engaged_seconds:,
+ time_to_first_interaction_ms:
+ )
+ return if session_id.blank?
+
+ session_id = session_id.slice(0, MAX_SESSION_ID_LENGTH)
+
+ upsert_all(
+ [
+ {
+ session_id:,
+ mouse_move_events:,
+ click_events:,
+ key_events:,
+ scroll_events:,
+ touch_events:,
+ back_forward_events:,
+ engaged_seconds:,
+ time_to_first_interaction_ms:,
+ },
+ ],
+ unique_by: :session_id,
+ on_duplicate: greatest_on_duplicate_clause,
+ record_timestamps: true,
+ )
+ end
+
+ def self.greatest_on_duplicate_clause
+ assignments =
+ GREATEST_COLUMNS.map do |column|
+ "#{column} = GREATEST(#{table_name}.#{column}, EXCLUDED.#{column})"
+ end
+ assignments << "updated_at = EXCLUDED.updated_at"
+ Arel.sql(assignments.join(", "))
+ end
+ private_class_method :greatest_on_duplicate_clause
+end
+
+# == Schema Information
+#
+# Table name: browser_pageview_session_engagements
+#
+# id :bigint not null, primary key
+# back_forward_events :integer default(0), not null
+# click_events :integer default(0), not null
+# engaged_seconds :integer default(0), not null
+# key_events :integer default(0), not null
+# mouse_move_events :integer default(0), not null
+# scroll_events :integer default(0), not null
+# time_to_first_interaction_ms :integer
+# touch_events :integer default(0), not null
+# created_at :datetime not null
+# updated_at :datetime not null
+# session_id :string(32) not null
+#
+# Indexes
+#
+# index_browser_pageview_session_engagements_on_created_at (created_at) USING brin
+# index_browser_pageview_session_engagements_on_session_id (session_id) UNIQUE
+#
diff --git a/config/site_settings.yml b/config/site_settings.yml
index f678b12b7c2..9596c3f38ae 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -629,6 +629,13 @@ basic:
min: 1
max: 20000
hidden: true
+ browser_pageview_max_engaged_seconds:
+ default: 1800
+ type: integer
+ min: 1
+ max: 86400
+ hidden: true
+ client: true
experimental_detect_crawler_pageviews:
default: false
hidden: true
diff --git a/db/migrate/20260623052745_create_browser_pageview_session_engagements.rb b/db/migrate/20260623052745_create_browser_pageview_session_engagements.rb
new file mode 100644
index 00000000000..15d013b9b06
--- /dev/null
+++ b/db/migrate/20260623052745_create_browser_pageview_session_engagements.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class CreateBrowserPageviewSessionEngagements < ActiveRecord::Migration[8.0]
+ def change
+ create_table :browser_pageview_session_engagements do |t|
+ t.string :session_id, null: false, limit: 32
+ t.integer :mouse_move_events, null: false, default: 0
+ t.integer :click_events, null: false, default: 0
+ t.integer :key_events, null: false, default: 0
+ t.integer :scroll_events, null: false, default: 0
+ t.integer :touch_events, null: false, default: 0
+ t.integer :back_forward_events, null: false, default: 0
+ t.integer :engaged_seconds, null: false, default: 0
+ t.integer :time_to_first_interaction_ms, null: true
+ t.timestamps
+ end
+
+ add_index :browser_pageview_session_engagements, :session_id, unique: true
+ add_index :browser_pageview_session_engagements, :created_at, using: :brin
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index c74b7888d2b..936a2d1496e 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -2048,6 +2048,45 @@ CREATE SEQUENCE public.browser_pageview_referrer_daily_rollups_id_seq
ALTER SEQUENCE public.browser_pageview_referrer_daily_rollups_id_seq OWNED BY public.browser_pageview_referrer_daily_rollups.id;
+--
+-- Name: browser_pageview_session_engagements; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.browser_pageview_session_engagements (
+ id bigint NOT NULL,
+ session_id character varying(32) NOT NULL,
+ mouse_move_events integer DEFAULT 0 NOT NULL,
+ click_events integer DEFAULT 0 NOT NULL,
+ key_events integer DEFAULT 0 NOT NULL,
+ scroll_events integer DEFAULT 0 NOT NULL,
+ touch_events integer DEFAULT 0 NOT NULL,
+ back_forward_events integer DEFAULT 0 NOT NULL,
+ engaged_seconds integer DEFAULT 0 NOT NULL,
+ time_to_first_interaction_ms integer,
+ created_at timestamp(6) without time zone NOT NULL,
+ updated_at timestamp(6) without time zone NOT NULL
+);
+
+
+--
+-- Name: browser_pageview_session_engagements_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.browser_pageview_session_engagements_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+--
+-- Name: browser_pageview_session_engagements_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.browser_pageview_session_engagements_id_seq OWNED BY public.browser_pageview_session_engagements.id;
+
+
--
-- Name: calendar_events; Type: TABLE; Schema: public; Owner: -
--
@@ -12329,6 +12368,13 @@ ALTER TABLE ONLY public.browser_pageview_events ALTER COLUMN id SET DEFAULT next
ALTER TABLE ONLY public.browser_pageview_referrer_daily_rollups ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_referrer_daily_rollups_id_seq'::regclass);
+--
+-- Name: browser_pageview_session_engagements id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.browser_pageview_session_engagements ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_session_engagements_id_seq'::regclass);
+
+
--
-- Name: calendar_events id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -14540,6 +14586,14 @@ ALTER TABLE ONLY public.browser_pageview_referrer_daily_rollups
ADD CONSTRAINT browser_pageview_referrer_daily_rollups_pkey PRIMARY KEY (id);
+--
+-- Name: browser_pageview_session_engagements browser_pageview_session_engagements_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.browser_pageview_session_engagements
+ ADD CONSTRAINT browser_pageview_session_engagements_pkey PRIMARY KEY (id);
+
+
--
-- Name: calendar_events calendar_events_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -18143,6 +18197,20 @@ CREATE INDEX index_browser_pageview_events_on_topic_id ON public.browser_pagevie
CREATE INDEX index_browser_pageview_events_on_user_id ON public.browser_pageview_events USING btree (user_id);
+--
+-- Name: index_browser_pageview_session_engagements_on_created_at; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_browser_pageview_session_engagements_on_created_at ON public.browser_pageview_session_engagements USING brin (created_at);
+
+
+--
+-- Name: index_browser_pageview_session_engagements_on_session_id; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX index_browser_pageview_session_engagements_on_session_id ON public.browser_pageview_session_engagements USING btree (session_id);
+
+
--
-- Name: index_calendar_events_on_post_id; Type: INDEX; Schema: public; Owner: -
--
@@ -22288,6 +22356,7 @@ INSERT INTO "schema_migrations" (version) VALUES
('20260626055145'),
('20260624140945'),
('20260623090824'),
+('20260623052745'),
('20260622201006'),
('20260622201005'),
('20260622140747'),
diff --git a/frontend/discourse/app/instance-initializers/detect-human-activity.js b/frontend/discourse/app/instance-initializers/detect-human-activity.js
new file mode 100644
index 00000000000..69acb165943
--- /dev/null
+++ b/frontend/discourse/app/instance-initializers/detect-human-activity.js
@@ -0,0 +1,14 @@
+export default {
+ after: "inject-objects",
+
+ initialize(owner) {
+ const enabled =
+ document.querySelector("meta[name=discourse-engagement-tracking-enabled]")
+ ?.content === "true";
+ if (!enabled) {
+ return;
+ }
+
+ owner.lookup("service:human-activity-tracker").start();
+ },
+};
diff --git a/frontend/discourse/app/services/human-activity-tracker.js b/frontend/discourse/app/services/human-activity-tracker.js
new file mode 100644
index 00000000000..8773bf6a26c
--- /dev/null
+++ b/frontend/discourse/app/services/human-activity-tracker.js
@@ -0,0 +1,215 @@
+import { cancel } from "@ember/runloop";
+import Service, { service } from "@ember/service";
+import getURL from "discourse/lib/get-url";
+import discourseLater from "discourse/lib/later";
+import {
+ browserAttention,
+ onBrowserAttentionChange,
+ removeOnBrowserAttentionChange,
+} from "discourse/lib/user-presence";
+
+const ENGAGEMENT_PATH = "/srv/se";
+
+const EVENT_COUNTERS = {
+ mousedown: "click_events",
+ keydown: "key_events",
+ touchstart: "touch_events",
+ scroll: "scroll_events",
+ popstate: "back_forward_events",
+};
+
+const MAX_HUMAN_STEP = 100;
+
+const FLUSH_DELAY_MS = 3 * 60 * 1000;
+
+const THROTTLE_MS = 3000;
+
+export default class HumanActivityTracker extends Service {
+ @service siteSettings;
+
+ now = () => performance.now();
+
+ transport = (body) => {
+ navigator.sendBeacon?.(
+ getURL(ENGAGEMENT_PATH),
+ new Blob([JSON.stringify(body)], { type: "application/json" })
+ );
+ };
+
+ scheduleFlush = (callback) => discourseLater(callback, FLUSH_DELAY_MS);
+
+ #sessionId;
+ #startedAt;
+ #firstInteractionAt = null;
+ #lastPosition = null;
+ #flushTimer = null;
+ #engagedMs = 0;
+ #engagedSince = null;
+ #lastSentMs = null;
+ #counts;
+ #activityListener;
+ #mouseMoveListener;
+ #attentionListener;
+ #pagehideListener;
+
+ willDestroy() {
+ super.willDestroy(...arguments);
+ this.stop();
+ }
+
+ start() {
+ this.#sessionId = document.querySelector(
+ "meta[name=discourse-track-view-session-id]"
+ )?.content;
+ if (!this.#sessionId) {
+ return;
+ }
+
+ this.#startedAt = this.now();
+ this.#counts = {
+ mouse_move_events: 0,
+ click_events: 0,
+ key_events: 0,
+ scroll_events: 0,
+ touch_events: 0,
+ back_forward_events: 0,
+ };
+
+ this.#activityListener = (event) => this.#handleActivity(event);
+ this.#mouseMoveListener = (event) => this.#handleMouseMove(event);
+ this.#attentionListener = (attention) =>
+ this.#handleAttentionChange(attention);
+ this.#pagehideListener = () => this.#flush({ force: true });
+
+ Object.keys(EVENT_COUNTERS).forEach((eventName) => {
+ window.addEventListener(eventName, this.#activityListener, {
+ passive: true,
+ });
+ });
+ window.addEventListener("mousemove", this.#mouseMoveListener, {
+ passive: true,
+ });
+ onBrowserAttentionChange(this.#attentionListener);
+ window.addEventListener("pagehide", this.#pagehideListener);
+
+ this.#handleAttentionChange(browserAttention());
+ this.#scheduleNextFlush();
+ }
+
+ #scheduleNextFlush() {
+ this.#flushTimer = this.scheduleFlush(() => {
+ this.#flush();
+ this.#scheduleNextFlush();
+ });
+ }
+
+ stop() {
+ if (this.#flushTimer) {
+ cancel(this.#flushTimer);
+ this.#flushTimer = null;
+ }
+
+ if (this.#activityListener) {
+ Object.keys(EVENT_COUNTERS).forEach((eventName) => {
+ window.removeEventListener(eventName, this.#activityListener);
+ });
+ this.#activityListener = null;
+ }
+ if (this.#mouseMoveListener) {
+ window.removeEventListener("mousemove", this.#mouseMoveListener);
+ this.#mouseMoveListener = null;
+ }
+ if (this.#attentionListener) {
+ removeOnBrowserAttentionChange(this.#attentionListener);
+ this.#attentionListener = null;
+ }
+ if (this.#pagehideListener) {
+ window.removeEventListener("pagehide", this.#pagehideListener);
+ this.#pagehideListener = null;
+ }
+ }
+
+ #handleActivity(event) {
+ this.#counts[EVENT_COUNTERS[event.type]]++;
+ this.#markInteraction();
+ }
+
+ #handleMouseMove(event) {
+ const position = { x: event.clientX, y: event.clientY };
+
+ if (this.#lastPosition) {
+ const distance = Math.hypot(
+ position.x - this.#lastPosition.x,
+ position.y - this.#lastPosition.y
+ );
+
+ if (distance > 0 && distance <= MAX_HUMAN_STEP) {
+ this.#counts.mouse_move_events++;
+ this.#markInteraction();
+ }
+ }
+
+ this.#lastPosition = position;
+ }
+
+ #handleAttentionChange({ focused, visible }) {
+ const engaged = focused && visible;
+
+ if (engaged && this.#engagedSince === null) {
+ this.#engagedSince = this.now();
+ } else if (!engaged && this.#engagedSince !== null) {
+ this.#engagedMs += this.now() - this.#engagedSince;
+ this.#engagedSince = null;
+ this.#flush();
+ }
+ }
+
+ #markInteraction() {
+ this.#firstInteractionAt ??= this.now();
+ }
+
+ #engagedSeconds() {
+ const live =
+ this.#engagedSince === null ? 0 : this.now() - this.#engagedSince;
+ return Math.min(
+ Math.floor((this.#engagedMs + live) / 1000),
+ this.siteSettings.browser_pageview_max_engaged_seconds
+ );
+ }
+
+ #buildPayload() {
+ return {
+ session_id: this.#sessionId,
+ ...this.#counts,
+ engaged_seconds: this.#engagedSeconds(),
+ time_to_first_interaction_ms:
+ this.#firstInteractionAt === null
+ ? null
+ : Math.round(this.#firstInteractionAt - this.#startedAt),
+ };
+ }
+
+ #flush({ force = false } = {}) {
+ if (force && this.#flushTimer) {
+ cancel(this.#flushTimer);
+ this.#flushTimer = null;
+ }
+
+ const total = Object.values(this.#counts).reduce((sum, n) => sum + n, 0);
+ if (total === 0) {
+ return;
+ }
+
+ const now = this.now();
+ if (
+ !force &&
+ this.#lastSentMs !== null &&
+ now - this.#lastSentMs < THROTTLE_MS
+ ) {
+ return;
+ }
+ this.#lastSentMs = now;
+
+ this.transport(this.#buildPayload());
+ }
+}
diff --git a/frontend/discourse/tests/unit/services/human-activity-tracker-test.js b/frontend/discourse/tests/unit/services/human-activity-tracker-test.js
new file mode 100644
index 00000000000..edb50d8fd2c
--- /dev/null
+++ b/frontend/discourse/tests/unit/services/human-activity-tracker-test.js
@@ -0,0 +1,190 @@
+import { setupTest } from "ember-qunit";
+import { module, test } from "qunit";
+import sinon from "sinon";
+import { processBrowserAttentionChange } from "discourse/lib/user-presence";
+
+const SESSION_ID = "S".repeat(32);
+
+function blur(context) {
+ context.focused = false;
+ processBrowserAttentionChange();
+}
+
+function focus(context) {
+ context.focused = true;
+ processBrowserAttentionChange();
+}
+
+function hide(context) {
+ context.visibility = "hidden";
+ processBrowserAttentionChange();
+}
+
+function pagehide() {
+ window.dispatchEvent(new Event("pagehide"));
+}
+
+module("Unit | Service | human-activity-tracker", function (hooks) {
+ setupTest(hooks);
+
+ hooks.beforeEach(function () {
+ this.clock = { ms: 0 };
+ this.focused = true;
+ this.visibility = "visible";
+ this.sent = [];
+
+ Object.defineProperty(document, "visibilityState", {
+ configurable: true,
+ get: () => this.visibility,
+ });
+ sinon.stub(document, "hasFocus").callsFake(() => this.focused);
+
+ this.meta = document.createElement("meta");
+ this.meta.name = "discourse-track-view-session-id";
+ this.meta.content = SESSION_ID;
+ document.head.appendChild(this.meta);
+
+ this.tracker = this.owner.lookup("service:human-activity-tracker");
+ this.tracker.now = () => this.clock.ms;
+ this.tracker.transport = (body) => this.sent.push(body);
+ this.tracker.scheduleFlush = (callback) => {
+ this.flushTick = callback;
+ return 1;
+ };
+ this.tracker.start();
+ });
+
+ hooks.afterEach(function () {
+ this.tracker?.stop();
+ this.meta.remove();
+ delete document.visibilityState;
+ });
+
+ test("sends nothing when there was no interaction", function (assert) {
+ pagehide();
+
+ assert.strictEqual(this.sent.length, 0);
+ });
+
+ test("counts interaction events and reports them on flush", function (assert) {
+ window.dispatchEvent(new KeyboardEvent("keydown"));
+ window.dispatchEvent(new MouseEvent("mousedown"));
+ window.dispatchEvent(new Event("scroll"));
+ pagehide();
+
+ const payload = this.sent.at(-1);
+ assert.strictEqual(payload.session_id, SESSION_ID);
+ assert.strictEqual(payload.key_events, 1);
+ assert.strictEqual(payload.click_events, 1);
+ assert.strictEqual(payload.scroll_events, 1);
+ });
+
+ test("counts continuous mouse movement but ignores teleporting jumps", function (assert) {
+ window.dispatchEvent(
+ new MouseEvent("mousemove", { clientX: 10, clientY: 10 })
+ );
+ // Within MAX_HUMAN_STEP of the previous point — counted.
+ window.dispatchEvent(
+ new MouseEvent("mousemove", { clientX: 30, clientY: 30 })
+ );
+ // A large jump — ignored.
+ window.dispatchEvent(
+ new MouseEvent("mousemove", { clientX: 900, clientY: 900 })
+ );
+ pagehide();
+
+ assert.strictEqual(this.sent.at(-1).mouse_move_events, 1);
+ });
+
+ test("reports the time to the first interaction", function (assert) {
+ this.clock.ms = 2500;
+ window.dispatchEvent(new KeyboardEvent("keydown"));
+
+ this.clock.ms = 9000;
+ pagehide();
+
+ assert.strictEqual(this.sent.at(-1).time_to_first_interaction_ms, 2500);
+ });
+
+ test("accumulates only visible-and-focused time as engaged duration", function (assert) {
+ window.dispatchEvent(new KeyboardEvent("keydown"));
+
+ this.clock.ms = 4000;
+ blur(this);
+
+ this.clock.ms = 10_000;
+ focus(this);
+
+ this.clock.ms = 13_000;
+ pagehide();
+
+ assert.strictEqual(this.sent.at(-1).engaged_seconds, 7);
+ });
+
+ test("flushes the latest snapshot when the tab is hidden", function (assert) {
+ window.dispatchEvent(new KeyboardEvent("keydown"));
+
+ this.clock.ms = 5000;
+ hide(this);
+
+ assert.strictEqual(this.sent.length, 1);
+ assert.strictEqual(this.sent.at(-1).engaged_seconds, 5);
+ });
+
+ test("caps engaged seconds at the configured maximum", function (assert) {
+ this.tracker.siteSettings.browser_pageview_max_engaged_seconds = 5;
+ window.dispatchEvent(new KeyboardEvent("keydown"));
+
+ this.clock.ms = 9000;
+ pagehide();
+
+ assert.strictEqual(this.sent.at(-1).engaged_seconds, 5);
+ });
+
+ test("throttles sends to at most one every three seconds", function (assert) {
+ window.dispatchEvent(new KeyboardEvent("keydown"));
+
+ this.clock.ms = 1000;
+ blur(this);
+
+ this.clock.ms = 2000;
+ focus(this);
+ this.clock.ms = 3000;
+ blur(this);
+
+ assert.strictEqual(this.sent.length, 1);
+
+ this.clock.ms = 6000;
+ focus(this);
+ this.clock.ms = 7000;
+ blur(this);
+
+ assert.strictEqual(this.sent.length, 2);
+ });
+
+ test("always flushes on pagehide, bypassing the throttle", function (assert) {
+ window.dispatchEvent(new KeyboardEvent("keydown"));
+
+ this.clock.ms = 1000;
+ blur(this);
+
+ this.clock.ms = 2000;
+ focus(this);
+ this.clock.ms = 3000;
+ pagehide();
+
+ assert.strictEqual(this.sent.length, 2);
+ });
+
+ test("keeps sending periodic snapshots on each flush cadence", function (assert) {
+ window.dispatchEvent(new KeyboardEvent("keydown"));
+
+ this.clock.ms = 180_000;
+ this.flushTick();
+
+ this.clock.ms = 360_000;
+ this.flushTick();
+
+ assert.strictEqual(this.sent.length, 2);
+ });
+});
diff --git a/lib/discourse.rb b/lib/discourse.rb
index fdb65ee8197..0c48ad46067 100644
--- a/lib/discourse.rb
+++ b/lib/discourse.rb
@@ -687,6 +687,10 @@ module Discourse
"#{Discourse.base_path}/srv/pv"
end
+ def self.engagement_tracking_path
+ "#{Discourse.base_path}/srv/se"
+ end
+
class << self
alias_method :base_url_no_path, :base_url_no_prefix
end
diff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rb
index 639ad29e6a5..5e09041ba58 100644
--- a/lib/middleware/request_tracker.rb
+++ b/lib/middleware/request_tracker.rb
@@ -349,7 +349,7 @@ class Middleware::RequestTracker
env["discourse.request_tracker"] = self
if self.class.is_beacon_tracking_request?(request)
- if self.class.same_origin_beacon_request?(request)
+ if self.class.same_origin_request?(request)
result = [204, {}, []]
else
env["discourse.request_tracker.skip"] = true
@@ -363,6 +363,16 @@ class Middleware::RequestTracker
return result
end
+ if self.class.is_engagement_tracking_request?(request)
+ env["discourse.request_tracker.skip"] = true
+ if self.class.same_origin_request?(request)
+ self.class.track_session_engagement(env)
+ return 204, {}, []
+ else
+ return 403, {}, []
+ end
+ end
+
MethodProfiler.start
if SiteSetting.instrument_gc_stat_per_request
@@ -664,7 +674,7 @@ class Middleware::RequestTracker
) && request.post? && request.path == Discourse.beacon_pv_tracking_path
end
- def self.same_origin_beacon_request?(request)
+ def self.same_origin_request?(request)
origin = request.get_header("HTTP_ORIGIN").presence || request.referer.presence
return false if origin.blank?
@@ -681,15 +691,52 @@ class Middleware::RequestTracker
request.post? && request.path == "#{Discourse.base_path}/pageview"
end
- def self.extract_beacon_view_tracking_data(env)
+ def self.is_engagement_tracking_request?(request)
+ UpcomingChanges.enabled?(:dashboard_improvements) &&
+ SiteSetting.persist_browser_pageview_events && request.post? &&
+ request.path == Discourse.engagement_tracking_path
+ end
+
+ def self.track_session_engagement(env)
+ payload = read_json_body(env)
+ return unless payload.is_a?(Hash)
+
+ Scheduler::Defer.later("Track session engagement") do
+ next if Discourse.pg_readonly_mode?
+
+ BrowserPageviewSessionEngagement.upsert_from_payload(
+ session_id: payload["session_id"],
+ mouse_move_events: payload["mouse_move_events"].to_i,
+ click_events: payload["click_events"].to_i,
+ key_events: payload["key_events"].to_i,
+ scroll_events: payload["scroll_events"].to_i,
+ touch_events: payload["touch_events"].to_i,
+ back_forward_events: payload["back_forward_events"].to_i,
+ engaged_seconds:
+ payload["engaged_seconds"].to_i.clamp(
+ 0,
+ SiteSetting.browser_pageview_max_engaged_seconds,
+ ),
+ time_to_first_interaction_ms: payload["time_to_first_interaction_ms"].presence&.to_i,
+ )
+ rescue => e
+ Rails.logger.warn("Discarding session engagement: #{e.message}")
+ end
+ end
+
+ def self.read_json_body(env)
body = env["rack.input"]&.read
env["rack.input"]&.rewind
- data =
- begin
- JSON.parse(body)
- rescue JSON::ParserError
- {}
- end
+ return if body.blank?
+
+ JSON.parse(body)
+ rescue JSON::ParserError
+ nil
+ end
+ private_class_method :read_json_body
+
+ def self.extract_beacon_view_tracking_data(env)
+ data = read_json_body(env) || {}
topic_id = data["topic_id"]&.to_i
tracking_url = data["url"]&.slice(0, MAX_URL_LENGTH)
diff --git a/spec/helpers/application_helper_spec.rb b/spec/helpers/application_helper_spec.rb
index 7e91f9b7535..5e27c5a7199 100644
--- a/spec/helpers/application_helper_spec.rb
+++ b/spec/helpers/application_helper_spec.rb
@@ -32,6 +32,26 @@ RSpec.describe ApplicationHelper do
expect(tags).to include('name="discourse-track-view-session-id"')
expect(tags).to include('name="discourse-beacon-pageview-enabled"')
end
+
+ it "includes the engagement tracking meta tag when pageview events are persisted" do
+ SiteSetting.dashboard_improvements = true
+ SiteSetting.persist_browser_pageview_events = true
+
+ tags = helper.discourse_pageview_tracking_meta_tags
+
+ expect(tags).to include('name="discourse-engagement-tracking-enabled"')
+ end
+
+ it "omits the engagement tracking meta tag in a trigger-only config so the browser does not send /srv/se beacons the server rejects" do
+ SiteSetting.dashboard_improvements = true
+ SiteSetting.persist_browser_pageview_events = false
+ SiteSetting.trigger_browser_pageview_events = true
+
+ tags = helper.discourse_pageview_tracking_meta_tags
+
+ expect(tags).to include('name="discourse-beacon-pageview-enabled"')
+ expect(tags).not_to include('name="discourse-engagement-tracking-enabled"')
+ end
end
describe "preload_script" do
diff --git a/spec/jobs/clean_up_browser_pageview_events_spec.rb b/spec/jobs/clean_up_browser_pageview_events_spec.rb
index 1242f870a41..f2b13a6efef 100644
--- a/spec/jobs/clean_up_browser_pageview_events_spec.rb
+++ b/spec/jobs/clean_up_browser_pageview_events_spec.rb
@@ -18,6 +18,17 @@ RSpec.describe Jobs::CleanUpBrowserPageviewEvents do
expect(BrowserPageviewEvent.all).to contain_exactly(fresh_event)
end
+ it "deletes session engagements older than 3 months and keeps fresher ones" do
+ SiteSetting.clean_up_browser_pageview_events = true
+ fresh = BrowserPageviewSessionEngagement.create!(session_id: "fresh", created_at: 1.month.ago)
+ BrowserPageviewSessionEngagement.create!(session_id: "old", created_at: 4.months.ago)
+
+ expect { described_class.new.execute({}) }.to change {
+ BrowserPageviewSessionEngagement.count
+ }.by(-1)
+ expect(BrowserPageviewSessionEngagement.all).to contain_exactly(fresh)
+ end
+
it "deletes scores for deleted events" do
SiteSetting.clean_up_browser_pageview_events = true
fresh_event = Fabricate(:browser_pageview_event, created_at: 1.month.ago)
diff --git a/spec/lib/middleware/request_tracker_spec.rb b/spec/lib/middleware/request_tracker_spec.rb
index 5858bc3cb1c..904d6499095 100644
--- a/spec/lib/middleware/request_tracker_spec.rb
+++ b/spec/lib/middleware/request_tracker_spec.rb
@@ -1959,4 +1959,172 @@ RSpec.describe Middleware::RequestTracker do
expect(fake_logger.warnings).to be_empty
end
end
+
+ describe "session engagement tracking via /srv/se" do
+ before do
+ SiteSetting.dashboard_improvements = true
+ SiteSetting.persist_browser_pageview_events = true
+ end
+
+ def engagement_env(body_hash, extra = {})
+ env(
+ {
+ :path => "/srv/se",
+ "HTTP_HOST" => "test.localhost",
+ "REQUEST_METHOD" => "POST",
+ "CONTENT_TYPE" => "application/json",
+ "rack.input" => StringIO.new(JSON.generate(body_hash)),
+ }.merge(extra),
+ )
+ end
+
+ let(:same_origin) { { "HTTP_ORIGIN" => "http://test.localhost" } }
+ let(:payload) do
+ {
+ session_id: "sess-1",
+ mouse_move_events: 12,
+ click_events: 3,
+ key_events: 5,
+ scroll_events: 7,
+ touch_events: 0,
+ back_forward_events: 1,
+ engaged_seconds: 420,
+ time_to_first_interaction_ms: 800,
+ }
+ end
+
+ it "returns 204 and inserts the row for a same-origin request" do
+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })
+
+ expect {
+ status, = middleware.call(engagement_env(payload, same_origin))
+ expect(status).to eq(204)
+ }.to change { BrowserPageviewSessionEngagement.count }.by(1)
+
+ row = BrowserPageviewSessionEngagement.find_by(session_id: "sess-1")
+ expect(row.mouse_move_events).to eq(12)
+ expect(row.back_forward_events).to eq(1)
+ expect(row.engaged_seconds).to eq(420)
+ expect(row.time_to_first_interaction_ms).to eq(800)
+ end
+
+ it "updates the existing row on a later snapshot for the same session" do
+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })
+ middleware.call(engagement_env(payload, same_origin))
+
+ later = payload.merge(mouse_move_events: 40, engaged_seconds: 900)
+
+ expect { middleware.call(engagement_env(later, same_origin)) }.not_to change {
+ BrowserPageviewSessionEngagement.count
+ }
+
+ row = BrowserPageviewSessionEngagement.find_by(session_id: "sess-1")
+ expect(row.mouse_move_events).to eq(40)
+ expect(row.engaged_seconds).to eq(900)
+ end
+
+ it "returns 403 and writes nothing for a cross-origin request" do
+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })
+
+ expect {
+ status, = middleware.call(engagement_env(payload, "HTTP_ORIGIN" => "https://evil.example"))
+ expect(status).to eq(403)
+ }.not_to change { BrowserPageviewSessionEngagement.count }
+ end
+
+ it "handles a malformed JSON body without writing a row" do
+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })
+
+ expect {
+ status, =
+ middleware.call(engagement_env({}, same_origin).merge("rack.input" => StringIO.new("x")))
+ expect(status).to eq(204)
+ }.not_to change { BrowserPageviewSessionEngagement.count }
+ end
+
+ it "ignores a parseable JSON body that is not an object" do
+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })
+
+ expect {
+ status, =
+ middleware.call(
+ engagement_env({}, same_origin).merge("rack.input" => StringIO.new("[1,2,3]")),
+ )
+ expect(status).to eq(204)
+ }.not_to change { BrowserPageviewSessionEngagement.count }
+ end
+
+ it "coerces string and float metric values to integers" do
+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })
+
+ middleware.call(
+ engagement_env(payload.merge(click_events: "9", engaged_seconds: 1234.9), same_origin),
+ )
+
+ expect(BrowserPageviewSessionEngagement.find_by(session_id: "sess-1")).to have_attributes(
+ click_events: 9,
+ engaged_seconds: 1234,
+ )
+ end
+
+ it "treats a null or blank time to first interaction as null rather than zero" do
+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })
+
+ [nil, ""].each do |value|
+ middleware.call(
+ engagement_env(payload.merge(time_to_first_interaction_ms: value), same_origin),
+ )
+
+ expect(
+ BrowserPageviewSessionEngagement.find_by(
+ session_id: "sess-1",
+ ).time_to_first_interaction_ms,
+ ).to be_nil
+ end
+ end
+
+ it "discards malformed-but-parseable payloads instead of raising in the deferred write" do
+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })
+
+ [
+ payload.merge(session_id: 123),
+ payload.merge(mouse_move_events: { "x" => 1 }),
+ payload.merge(mouse_move_events: 9_999_999_999),
+ ].each do |malformed|
+ expect {
+ status, = middleware.call(engagement_env(malformed, same_origin))
+ expect(status).to eq(204)
+ }.not_to change { BrowserPageviewSessionEngagement.count }
+ end
+ end
+
+ it "clamps engaged seconds to the configured maximum" do
+ SiteSetting.browser_pageview_max_engaged_seconds = 60
+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })
+
+ middleware.call(engagement_env(payload.merge(engaged_seconds: 5000), same_origin))
+
+ expect(BrowserPageviewSessionEngagement.find_by(session_id: "sess-1").engaged_seconds).to eq(
+ 60,
+ )
+ end
+
+ it "writes nothing when the database is in readonly mode" do
+ Discourse.stubs(:pg_readonly_mode?).returns(true)
+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })
+
+ expect {
+ status, = middleware.call(engagement_env(payload, same_origin))
+ expect(status).to eq(204)
+ }.not_to change { BrowserPageviewSessionEngagement.count }
+ end
+
+ it "does not intercept the request when dashboard_improvements is disabled" do
+ SiteSetting.dashboard_improvements = false
+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })
+
+ status, = middleware.call(engagement_env(payload, same_origin))
+ expect(status).to eq(200)
+ end
+ end
end
diff --git a/spec/models/browser_pageview_session_engagement_spec.rb b/spec/models/browser_pageview_session_engagement_spec.rb
new file mode 100644
index 00000000000..f52a7f1d210
--- /dev/null
+++ b/spec/models/browser_pageview_session_engagement_spec.rb
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+RSpec.describe BrowserPageviewSessionEngagement do
+ describe ".upsert_from_payload" do
+ let(:attributes) do
+ {
+ session_id: "sess-1",
+ mouse_move_events: 12,
+ click_events: 3,
+ key_events: 5,
+ scroll_events: 7,
+ touch_events: 0,
+ back_forward_events: 1,
+ engaged_seconds: 4200,
+ time_to_first_interaction_ms: 800,
+ }
+ end
+
+ it "inserts a row with the given metrics" do
+ expect { described_class.upsert_from_payload(**attributes) }.to change {
+ described_class.count
+ }.by(1)
+
+ expect(described_class.find_by(session_id: "sess-1")).to have_attributes(
+ mouse_move_events: 12,
+ click_events: 3,
+ key_events: 5,
+ scroll_events: 7,
+ touch_events: 0,
+ back_forward_events: 1,
+ engaged_seconds: 4200,
+ time_to_first_interaction_ms: 800,
+ )
+ end
+
+ it "stores a null time to first interaction when none is reported" do
+ described_class.upsert_from_payload(**attributes.merge(time_to_first_interaction_ms: nil))
+
+ expect(described_class.find_by(session_id: "sess-1").time_to_first_interaction_ms).to be_nil
+ end
+
+ it "truncates an over-long session id to the column limit" do
+ long_id = "a" * (described_class::MAX_SESSION_ID_LENGTH + 10)
+
+ described_class.upsert_from_payload(**attributes.merge(session_id: long_id))
+
+ expect(described_class.first.session_id).to eq("a" * described_class::MAX_SESSION_ID_LENGTH)
+ end
+
+ it "writes nothing when the session id is blank" do
+ expect {
+ described_class.upsert_from_payload(**attributes.merge(session_id: ""))
+ described_class.upsert_from_payload(**attributes.merge(session_id: nil))
+ }.not_to change { described_class.count }
+ end
+
+ it "updates the existing row for the same session instead of duplicating" do
+ described_class.upsert_from_payload(**attributes)
+
+ expect {
+ described_class.upsert_from_payload(
+ **attributes.merge(mouse_move_events: 40, engaged_seconds: 9000),
+ )
+ }.not_to change { described_class.count }
+
+ expect(described_class.find_by(session_id: "sess-1")).to have_attributes(
+ mouse_move_events: 40,
+ engaged_seconds: 9000,
+ )
+ end
+
+ it "keeps the largest value per column across snapshots" do
+ described_class.upsert_from_payload(
+ **attributes.merge(mouse_move_events: 40, click_events: 3),
+ )
+
+ described_class.upsert_from_payload(
+ **attributes.merge(mouse_move_events: 12, click_events: 20),
+ )
+
+ expect(described_class.find_by(session_id: "sess-1")).to have_attributes(
+ mouse_move_events: 40,
+ click_events: 20,
+ )
+ end
+ end
+end
diff --git a/spec/system/detect_human_activity_spec.rb b/spec/system/detect_human_activity_spec.rb
new file mode 100644
index 00000000000..ebc1842c237
--- /dev/null
+++ b/spec/system/detect_human_activity_spec.rb
@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+describe "Detect human activity" do
+ fab!(:topics) { Fabricate.times(5, :post).map(&:topic) }
+ let(:discovery) { PageObjects::Pages::Discovery.new }
+
+ before do
+ SiteSetting.dashboard_improvements = true
+ SiteSetting.persist_browser_pageview_events = true
+ end
+
+ def flush_engagement
+ page.driver.with_playwright_page do |pw_page|
+ pw_page.evaluate("window.dispatchEvent(new Event('pagehide'))")
+ end
+ end
+
+ def engagement
+ BrowserPageviewSessionEngagement.first
+ end
+
+ it "stores continuous mouse movement and time to first interaction" do
+ visit("/")
+ expect(discovery.topic_list).to have_topics(count: 5)
+
+ # `steps` interpolates the movement into many small mousemove events,
+ # mimicking the continuous path a real pointer traces.
+ page.driver.with_playwright_page { |pw_page| pw_page.mouse.move(400, 400, steps: 30) }
+ flush_engagement
+
+ try_until_success { expect(BrowserPageviewSessionEngagement.count).to eq(1) }
+ expect(engagement.mouse_move_events).to be > 0
+ expect(engagement.time_to_first_interaction_ms).to be_present
+ end
+
+ it "ignores teleporting mouse jumps" do
+ visit("/")
+ expect(discovery.topic_list).to have_topics(count: 5)
+
+ # Each move jumps straight to its destination with no intermediate events,
+ # the way a script driving the cursor would. A real keypress is added so a
+ # snapshot is actually sent (teleports alone leave the session empty).
+ page.driver.with_playwright_page do |pw_page|
+ pw_page.mouse.move(50, 50)
+ pw_page.mouse.move(600, 600)
+ pw_page.mouse.move(50, 600)
+ pw_page.keyboard.press("a")
+ end
+ flush_engagement
+
+ try_until_success { expect(BrowserPageviewSessionEngagement.count).to eq(1) }
+ expect(engagement.mouse_move_events).to eq(0)
+ expect(engagement.key_events).to eq(1)
+ end
+
+ it "accumulates engaged duration while the tab is visible and focused" do
+ visit("/")
+ expect(discovery.topic_list).to have_topics(count: 5)
+
+ # An interaction is needed so a snapshot is sent; then stay engaged for a
+ # beat before flushing so the duration has time to accumulate.
+ page.driver.with_playwright_page do |pw_page|
+ pw_page.keyboard.press("a")
+ pw_page.wait_for_timeout(1200)
+ end
+ flush_engagement
+
+ try_until_success { expect(BrowserPageviewSessionEngagement.count).to eq(1) }
+ expect(engagement.engaged_seconds).to be >= 1
+ end
+
+ it "counts back/forward navigation" do
+ visit("/")
+ expect(discovery.topic_list).to have_topics(count: 5)
+
+ # In-app navigation uses pushState (no popstate); the back button does fire
+ # popstate, which is what we count.
+ discovery.topic_list.visit_topic(topics[0])
+ page.go_back
+ flush_engagement
+
+ try_until_success { expect(BrowserPageviewSessionEngagement.count).to eq(1) }
+ expect(engagement.back_forward_events).to eq(1)
+ end
+end