A platform for community discussion. Free, open, simple.
DEV: Track browser session engagement for crawler detection (#41215)
Krzysztof Kotlarekd570c043029a
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rbindex 7df092f4dd1..28009aae3f3 100644--- a/app/helpers/application_helper.rb+++ b/app/helpers/application_helper.rb@@ -452,6 +452,11 @@ module ApplicationHelperif UpcomingChanges.enabled?(:dashboard_improvements)tags << tag.meta(name: "discourse-beacon-pageview-enabled", content: "true")end++ if UpcomingChanges.enabled?(:dashboard_improvements) &&+ SiteSetting.persist_browser_pageview_events+ tags << tag.meta(name: "discourse-engagement-tracking-enabled", content: "true")+ endtags.html_safeenddiff --git a/app/jobs/scheduled/clean_up_browser_pageview_events.rb b/app/jobs/scheduled/clean_up_browser_pageview_events.rbindex 12ed20182f0..e3f741013ce 100644--- a/app/jobs/scheduled/clean_up_browser_pageview_events.rb+++ b/app/jobs/scheduled/clean_up_browser_pageview_events.rb@@ -4,15 +4,23 @@ module Jobsclass CleanUpBrowserPageviewEvents < ::Jobs::Scheduledevery 1.day+ BATCH_SIZE = 10_000+def execute(args)return if !SiteSetting.clean_up_browser_pageview_events+ cutoff = BrowserPageviewEvent.retention_cutoff+BrowserPageviewEvent- .where("created_at < ?", BrowserPageviewEvent.retention_cutoff)- .in_batches(of: 10_000) do |browser_pageview_events|+ .where("created_at < ?", cutoff)+ .in_batches(of: BATCH_SIZE) do |browser_pageview_events|BrowserPageviewEventScore.where(event_id: browser_pageview_events.select(:id)).delete_allbrowser_pageview_events.delete_allend++ BrowserPageviewSessionEngagement+ .where("created_at < ?", cutoff)+ .in_batches(of: BATCH_SIZE) { |session_engagements| session_engagements.delete_all }endendenddiff --git a/app/models/browser_pageview_session_engagement.rb b/app/models/browser_pageview_session_engagement.rbnew file mode 100644index 00000000000..487540c5d86--- /dev/null+++ b/app/models/browser_pageview_session_engagement.rb@@ -0,0 +1,84 @@+# frozen_string_literal: true++class BrowserPageviewSessionEngagement < ActiveRecord::Base+ MAX_SESSION_ID_LENGTH = 32++ GREATEST_COLUMNS = %i[+ mouse_move_events+ click_events+ key_events+ scroll_events+ touch_events+ back_forward_events+ engaged_seconds+ time_to_first_interaction_ms+ ]++ def self.upsert_from_payload(+ session_id:,+ mouse_move_events:,+ click_events:,+ key_events:,+ scroll_events:,+ touch_events:,+ back_forward_events:,+ engaged_seconds:,+ time_to_first_interaction_ms:+ )+ return if session_id.blank?++ session_id = session_id.slice(0, MAX_SESSION_ID_LENGTH)++ upsert_all(+ [+ {+ session_id:,+ mouse_move_events:,+ click_events:,+ key_events:,+ scroll_events:,+ touch_events:,+ back_forward_events:,+ engaged_seconds:,+ time_to_first_interaction_ms:,+ },+ ],+ unique_by: :session_id,+ on_duplicate: greatest_on_duplicate_clause,+ record_timestamps: true,+ )+ end++ def self.greatest_on_duplicate_clause+ assignments =+ GREATEST_COLUMNS.map do |column|+ "#{column} = GREATEST(#{table_name}.#{column}, EXCLUDED.#{column})"+ end+ assignments << "updated_at = EXCLUDED.updated_at"+ Arel.sql(assignments.join(", "))+ end+ private_class_method :greatest_on_duplicate_clause+end++# == Schema Information+#+# Table name: browser_pageview_session_engagements+#+# id :bigint not null, primary key+# back_forward_events :integer default(0), not null+# click_events :integer default(0), not null+# engaged_seconds :integer default(0), not null+# key_events :integer default(0), not null+# mouse_move_events :integer default(0), not null+# scroll_events :integer default(0), not null+# time_to_first_interaction_ms :integer+# touch_events :integer default(0), not null+# created_at :datetime not null+# updated_at :datetime not null+# session_id :string(32) not null+#+# Indexes+#+# index_browser_pageview_session_engagements_on_created_at (created_at) USING brin+# index_browser_pageview_session_engagements_on_session_id (session_id) UNIQUE+#diff --git a/config/site_settings.yml b/config/site_settings.ymlindex f678b12b7c2..9596c3f38ae 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -629,6 +629,13 @@ basic:min: 1max: 20000hidden: true+ browser_pageview_max_engaged_seconds:+ default: 1800+ type: integer+ min: 1+ max: 86400+ hidden: true+ client: trueexperimental_detect_crawler_pageviews:default: falsehidden: truediff --git a/db/migrate/20260623052745_create_browser_pageview_session_engagements.rb b/db/migrate/20260623052745_create_browser_pageview_session_engagements.rbnew file mode 100644index 00000000000..15d013b9b06--- /dev/null+++ b/db/migrate/20260623052745_create_browser_pageview_session_engagements.rb@@ -0,0 +1,21 @@+# frozen_string_literal: true++class CreateBrowserPageviewSessionEngagements < ActiveRecord::Migration[8.0]+ def change+ create_table :browser_pageview_session_engagements do |t|+ t.string :session_id, null: false, limit: 32+ t.integer :mouse_move_events, null: false, default: 0+ t.integer :click_events, null: false, default: 0+ t.integer :key_events, null: false, default: 0+ t.integer :scroll_events, null: false, default: 0+ t.integer :touch_events, null: false, default: 0+ t.integer :back_forward_events, null: false, default: 0+ t.integer :engaged_seconds, null: false, default: 0+ t.integer :time_to_first_interaction_ms, null: true+ t.timestamps+ end++ add_index :browser_pageview_session_engagements, :session_id, unique: true+ add_index :browser_pageview_session_engagements, :created_at, using: :brin+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex c74b7888d2b..936a2d1496e 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -2048,6 +2048,45 @@ CREATE SEQUENCE public.browser_pageview_referrer_daily_rollups_id_seqALTER SEQUENCE public.browser_pageview_referrer_daily_rollups_id_seq OWNED BY public.browser_pageview_referrer_daily_rollups.id;+--+-- Name: browser_pageview_session_engagements; Type: TABLE; Schema: public; Owner: -+--++CREATE TABLE public.browser_pageview_session_engagements (+ id bigint NOT NULL,+ session_id character varying(32) NOT NULL,+ mouse_move_events integer DEFAULT 0 NOT NULL,+ click_events integer DEFAULT 0 NOT NULL,+ key_events integer DEFAULT 0 NOT NULL,+ scroll_events integer DEFAULT 0 NOT NULL,+ touch_events integer DEFAULT 0 NOT NULL,+ back_forward_events integer DEFAULT 0 NOT NULL,+ engaged_seconds integer DEFAULT 0 NOT NULL,+ time_to_first_interaction_ms integer,+ created_at timestamp(6) without time zone NOT NULL,+ updated_at timestamp(6) without time zone NOT NULL+);+++--+-- Name: browser_pageview_session_engagements_id_seq; Type: SEQUENCE; Schema: public; Owner: -+--++CREATE SEQUENCE public.browser_pageview_session_engagements_id_seq+ START WITH 1+ INCREMENT BY 1+ NO MINVALUE+ NO MAXVALUE+ CACHE 1;+++--+-- Name: browser_pageview_session_engagements_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -+--++ALTER SEQUENCE public.browser_pageview_session_engagements_id_seq OWNED BY public.browser_pageview_session_engagements.id;++---- Name: calendar_events; Type: TABLE; Schema: public; Owner: ---@@ -12329,6 +12368,13 @@ ALTER TABLE ONLY public.browser_pageview_events ALTER COLUMN id SET DEFAULT nextALTER TABLE ONLY public.browser_pageview_referrer_daily_rollups ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_referrer_daily_rollups_id_seq'::regclass);+--+-- Name: browser_pageview_session_engagements id; Type: DEFAULT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.browser_pageview_session_engagements ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_session_engagements_id_seq'::regclass);++---- Name: calendar_events id; Type: DEFAULT; Schema: public; Owner: ---@@ -14540,6 +14586,14 @@ ALTER TABLE ONLY public.browser_pageview_referrer_daily_rollupsADD CONSTRAINT browser_pageview_referrer_daily_rollups_pkey PRIMARY KEY (id);+--+-- Name: browser_pageview_session_engagements browser_pageview_session_engagements_pkey; Type: CONSTRAINT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.browser_pageview_session_engagements+ ADD CONSTRAINT browser_pageview_session_engagements_pkey PRIMARY KEY (id);++---- Name: calendar_events calendar_events_pkey; Type: CONSTRAINT; Schema: public; Owner: ---@@ -18143,6 +18197,20 @@ CREATE INDEX index_browser_pageview_events_on_topic_id ON public.browser_pagevieCREATE INDEX index_browser_pageview_events_on_user_id ON public.browser_pageview_events USING btree (user_id);+--+-- Name: index_browser_pageview_session_engagements_on_created_at; Type: INDEX; Schema: public; Owner: -+--++CREATE INDEX index_browser_pageview_session_engagements_on_created_at ON public.browser_pageview_session_engagements USING brin (created_at);+++--+-- Name: index_browser_pageview_session_engagements_on_session_id; Type: INDEX; Schema: public; Owner: -+--++CREATE UNIQUE INDEX index_browser_pageview_session_engagements_on_session_id ON public.browser_pageview_session_engagements USING btree (session_id);++---- Name: index_calendar_events_on_post_id; Type: INDEX; Schema: public; Owner: ---@@ -22288,6 +22356,7 @@ INSERT INTO "schema_migrations" (version) VALUES('20260626055145'),('20260624140945'),('20260623090824'),+('20260623052745'),('20260622201006'),('20260622201005'),('20260622140747'),diff --git a/frontend/discourse/app/instance-initializers/detect-human-activity.js b/frontend/discourse/app/instance-initializers/detect-human-activity.jsnew file mode 100644index 00000000000..69acb165943--- /dev/null+++ b/frontend/discourse/app/instance-initializers/detect-human-activity.js@@ -0,0 +1,14 @@+export default {+ after: "inject-objects",++ initialize(owner) {+ const enabled =+ document.querySelector("meta[name=discourse-engagement-tracking-enabled]")+ ?.content === "true";+ if (!enabled) {+ return;+ }++ owner.lookup("service:human-activity-tracker").start();+ },+};diff --git a/frontend/discourse/app/services/human-activity-tracker.js b/frontend/discourse/app/services/human-activity-tracker.jsnew file mode 100644index 00000000000..8773bf6a26c--- /dev/null+++ b/frontend/discourse/app/services/human-activity-tracker.js@@ -0,0 +1,215 @@+import { cancel } from "@ember/runloop";+import Service, { service } from "@ember/service";+import getURL from "discourse/lib/get-url";+import discourseLater from "discourse/lib/later";+import {+ browserAttention,+ onBrowserAttentionChange,+ removeOnBrowserAttentionChange,+} from "discourse/lib/user-presence";++const ENGAGEMENT_PATH = "/srv/se";++const EVENT_COUNTERS = {+ mousedown: "click_events",+ keydown: "key_events",+ touchstart: "touch_events",+ scroll: "scroll_events",+ popstate: "back_forward_events",+};++const MAX_HUMAN_STEP = 100;++const FLUSH_DELAY_MS = 3 * 60 * 1000;++const THROTTLE_MS = 3000;++export default class HumanActivityTracker extends Service {+ @service siteSettings;++ now = () => performance.now();++ transport = (body) => {+ navigator.sendBeacon?.(+ getURL(ENGAGEMENT_PATH),+ new Blob([JSON.stringify(body)], { type: "application/json" })+ );+ };++ scheduleFlush = (callback) => discourseLater(callback, FLUSH_DELAY_MS);++ #sessionId;+ #startedAt;+ #firstInteractionAt = null;+ #lastPosition = null;+ #flushTimer = null;+ #engagedMs = 0;+ #engagedSince = null;+ #lastSentMs = null;+ #counts;+ #activityListener;+ #mouseMoveListener;+ #attentionListener;+ #pagehideListener;++ willDestroy() {+ super.willDestroy(...arguments);+ this.stop();+ }++ start() {+ this.#sessionId = document.querySelector(+ "meta[name=discourse-track-view-session-id]"+ )?.content;+ if (!this.#sessionId) {+ return;+ }++ this.#startedAt = this.now();+ this.#counts = {+ mouse_move_events: 0,+ click_events: 0,+ key_events: 0,+ scroll_events: 0,+ touch_events: 0,+ back_forward_events: 0,+ };++ this.#activityListener = (event) => this.#handleActivity(event);+ this.#mouseMoveListener = (event) => this.#handleMouseMove(event);+ this.#attentionListener = (attention) =>+ this.#handleAttentionChange(attention);+ this.#pagehideListener = () => this.#flush({ force: true });++ Object.keys(EVENT_COUNTERS).forEach((eventName) => {+ window.addEventListener(eventName, this.#activityListener, {+ passive: true,+ });+ });+ window.addEventListener("mousemove", this.#mouseMoveListener, {+ passive: true,+ });+ onBrowserAttentionChange(this.#attentionListener);+ window.addEventListener("pagehide", this.#pagehideListener);++ this.#handleAttentionChange(browserAttention());+ this.#scheduleNextFlush();+ }++ #scheduleNextFlush() {+ this.#flushTimer = this.scheduleFlush(() => {+ this.#flush();+ this.#scheduleNextFlush();+ });+ }++ stop() {+ if (this.#flushTimer) {+ cancel(this.#flushTimer);+ this.#flushTimer = null;+ }++ if (this.#activityListener) {+ Object.keys(EVENT_COUNTERS).forEach((eventName) => {+ window.removeEventListener(eventName, this.#activityListener);+ });+ this.#activityListener = null;+ }+ if (this.#mouseMoveListener) {+ window.removeEventListener("mousemove", this.#mouseMoveListener);+ this.#mouseMoveListener = null;+ }+ if (this.#attentionListener) {+ removeOnBrowserAttentionChange(this.#attentionListener);+ this.#attentionListener = null;+ }+ if (this.#pagehideListener) {+ window.removeEventListener("pagehide", this.#pagehideListener);+ this.#pagehideListener = null;+ }+ }++ #handleActivity(event) {+ this.#counts[EVENT_COUNTERS[event.type]]++;+ this.#markInteraction();+ }++ #handleMouseMove(event) {+ const position = { x: event.clientX, y: event.clientY };++ if (this.#lastPosition) {+ const distance = Math.hypot(+ position.x - this.#lastPosition.x,+ position.y - this.#lastPosition.y+ );++ if (distance > 0 && distance <= MAX_HUMAN_STEP) {+ this.#counts.mouse_move_events++;+ this.#markInteraction();+ }+ }++ this.#lastPosition = position;+ }++ #handleAttentionChange({ focused, visible }) {+ const engaged = focused && visible;++ if (engaged && this.#engagedSince === null) {+ this.#engagedSince = this.now();+ } else if (!engaged && this.#engagedSince !== null) {+ this.#engagedMs += this.now() - this.#engagedSince;+ this.#engagedSince = null;+ this.#flush();+ }+ }++ #markInteraction() {+ this.#firstInteractionAt ??= this.now();+ }++ #engagedSeconds() {+ const live =+ this.#engagedSince === null ? 0 : this.now() - this.#engagedSince;+ return Math.min(+ Math.floor((this.#engagedMs + live) / 1000),+ this.siteSettings.browser_pageview_max_engaged_seconds+ );+ }++ #buildPayload() {+ return {+ session_id: this.#sessionId,+ ...this.#counts,+ engaged_seconds: this.#engagedSeconds(),+ time_to_first_interaction_ms:+ this.#firstInteractionAt === null+ ? null+ : Math.round(this.#firstInteractionAt - this.#startedAt),+ };+ }++ #flush({ force = false } = {}) {+ if (force && this.#flushTimer) {+ cancel(this.#flushTimer);+ this.#flushTimer = null;+ }++ const total = Object.values(this.#counts).reduce((sum, n) => sum + n, 0);+ if (total === 0) {+ return;+ }++ const now = this.now();+ if (+ !force &&+ this.#lastSentMs !== null &&+ now - this.#lastSentMs < THROTTLE_MS+ ) {+ return;+ }+ this.#lastSentMs = now;++ this.transport(this.#buildPayload());+ }+}diff --git a/frontend/discourse/tests/unit/services/human-activity-tracker-test.js b/frontend/discourse/tests/unit/services/human-activity-tracker-test.jsnew file mode 100644index 00000000000..edb50d8fd2c--- /dev/null+++ b/frontend/discourse/tests/unit/services/human-activity-tracker-test.js@@ -0,0 +1,190 @@+import { setupTest } from "ember-qunit";+import { module, test } from "qunit";+import sinon from "sinon";+import { processBrowserAttentionChange } from "discourse/lib/user-presence";++const SESSION_ID = "S".repeat(32);++function blur(context) {+ context.focused = false;+ processBrowserAttentionChange();+}++function focus(context) {+ context.focused = true;+ processBrowserAttentionChange();+}++function hide(context) {+ context.visibility = "hidden";+ processBrowserAttentionChange();+}++function pagehide() {+ window.dispatchEvent(new Event("pagehide"));+}++module("Unit | Service | human-activity-tracker", function (hooks) {+ setupTest(hooks);++ hooks.beforeEach(function () {+ this.clock = { ms: 0 };+ this.focused = true;+ this.visibility = "visible";+ this.sent = [];++ Object.defineProperty(document, "visibilityState", {+ configurable: true,+ get: () => this.visibility,+ });+ sinon.stub(document, "hasFocus").callsFake(() => this.focused);++ this.meta = document.createElement("meta");+ this.meta.name = "discourse-track-view-session-id";+ this.meta.content = SESSION_ID;+ document.head.appendChild(this.meta);++ this.tracker = this.owner.lookup("service:human-activity-tracker");+ this.tracker.now = () => this.clock.ms;+ this.tracker.transport = (body) => this.sent.push(body);+ this.tracker.scheduleFlush = (callback) => {+ this.flushTick = callback;+ return 1;+ };+ this.tracker.start();+ });++ hooks.afterEach(function () {+ this.tracker?.stop();+ this.meta.remove();+ delete document.visibilityState;+ });++ test("sends nothing when there was no interaction", function (assert) {+ pagehide();++ assert.strictEqual(this.sent.length, 0);+ });++ test("counts interaction events and reports them on flush", function (assert) {+ window.dispatchEvent(new KeyboardEvent("keydown"));+ window.dispatchEvent(new MouseEvent("mousedown"));+ window.dispatchEvent(new Event("scroll"));+ pagehide();++ const payload = this.sent.at(-1);+ assert.strictEqual(payload.session_id, SESSION_ID);+ assert.strictEqual(payload.key_events, 1);+ assert.strictEqual(payload.click_events, 1);+ assert.strictEqual(payload.scroll_events, 1);+ });++ test("counts continuous mouse movement but ignores teleporting jumps", function (assert) {+ window.dispatchEvent(+ new MouseEvent("mousemove", { clientX: 10, clientY: 10 })+ );+ // Within MAX_HUMAN_STEP of the previous point — counted.+ window.dispatchEvent(+ new MouseEvent("mousemove", { clientX: 30, clientY: 30 })+ );+ // A large jump — ignored.+ window.dispatchEvent(+ new MouseEvent("mousemove", { clientX: 900, clientY: 900 })+ );+ pagehide();++ assert.strictEqual(this.sent.at(-1).mouse_move_events, 1);+ });++ test("reports the time to the first interaction", function (assert) {+ this.clock.ms = 2500;+ window.dispatchEvent(new KeyboardEvent("keydown"));++ this.clock.ms = 9000;+ pagehide();++ assert.strictEqual(this.sent.at(-1).time_to_first_interaction_ms, 2500);+ });++ test("accumulates only visible-and-focused time as engaged duration", function (assert) {+ window.dispatchEvent(new KeyboardEvent("keydown"));++ this.clock.ms = 4000;+ blur(this);++ this.clock.ms = 10_000;+ focus(this);++ this.clock.ms = 13_000;+ pagehide();++ assert.strictEqual(this.sent.at(-1).engaged_seconds, 7);+ });++ test("flushes the latest snapshot when the tab is hidden", function (assert) {+ window.dispatchEvent(new KeyboardEvent("keydown"));++ this.clock.ms = 5000;+ hide(this);++ assert.strictEqual(this.sent.length, 1);+ assert.strictEqual(this.sent.at(-1).engaged_seconds, 5);+ });++ test("caps engaged seconds at the configured maximum", function (assert) {+ this.tracker.siteSettings.browser_pageview_max_engaged_seconds = 5;+ window.dispatchEvent(new KeyboardEvent("keydown"));++ this.clock.ms = 9000;+ pagehide();++ assert.strictEqual(this.sent.at(-1).engaged_seconds, 5);+ });++ test("throttles sends to at most one every three seconds", function (assert) {+ window.dispatchEvent(new KeyboardEvent("keydown"));++ this.clock.ms = 1000;+ blur(this);++ this.clock.ms = 2000;+ focus(this);+ this.clock.ms = 3000;+ blur(this);++ assert.strictEqual(this.sent.length, 1);++ this.clock.ms = 6000;+ focus(this);+ this.clock.ms = 7000;+ blur(this);++ assert.strictEqual(this.sent.length, 2);+ });++ test("always flushes on pagehide, bypassing the throttle", function (assert) {+ window.dispatchEvent(new KeyboardEvent("keydown"));++ this.clock.ms = 1000;+ blur(this);++ this.clock.ms = 2000;+ focus(this);+ this.clock.ms = 3000;+ pagehide();++ assert.strictEqual(this.sent.length, 2);+ });++ test("keeps sending periodic snapshots on each flush cadence", function (assert) {+ window.dispatchEvent(new KeyboardEvent("keydown"));++ this.clock.ms = 180_000;+ this.flushTick();++ this.clock.ms = 360_000;+ this.flushTick();++ assert.strictEqual(this.sent.length, 2);+ });+});diff --git a/lib/discourse.rb b/lib/discourse.rbindex fdb65ee8197..0c48ad46067 100644--- a/lib/discourse.rb+++ b/lib/discourse.rb@@ -687,6 +687,10 @@ module Discourse"#{Discourse.base_path}/srv/pv"end+ def self.engagement_tracking_path+ "#{Discourse.base_path}/srv/se"+ end+class << selfalias_method :base_url_no_path, :base_url_no_prefixenddiff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rbindex 639ad29e6a5..5e09041ba58 100644--- a/lib/middleware/request_tracker.rb+++ b/lib/middleware/request_tracker.rb@@ -349,7 +349,7 @@ class Middleware::RequestTrackerenv["discourse.request_tracker"] = selfif self.class.is_beacon_tracking_request?(request)- if self.class.same_origin_beacon_request?(request)+ if self.class.same_origin_request?(request)result = [204, {}, []]elseenv["discourse.request_tracker.skip"] = true@@ -363,6 +363,16 @@ class Middleware::RequestTrackerreturn resultend+ if self.class.is_engagement_tracking_request?(request)+ env["discourse.request_tracker.skip"] = true+ if self.class.same_origin_request?(request)+ self.class.track_session_engagement(env)+ return 204, {}, []+ else+ return 403, {}, []+ end+ end+MethodProfiler.startif SiteSetting.instrument_gc_stat_per_request@@ -664,7 +674,7 @@ class Middleware::RequestTracker) && request.post? && request.path == Discourse.beacon_pv_tracking_pathend- def self.same_origin_beacon_request?(request)+ def self.same_origin_request?(request)origin = request.get_header("HTTP_ORIGIN").presence || request.referer.presencereturn false if origin.blank?@@ -681,15 +691,52 @@ class Middleware::RequestTrackerrequest.post? && request.path == "#{Discourse.base_path}/pageview"end- def self.extract_beacon_view_tracking_data(env)+ def self.is_engagement_tracking_request?(request)+ UpcomingChanges.enabled?(:dashboard_improvements) &&+ SiteSetting.persist_browser_pageview_events && request.post? &&+ request.path == Discourse.engagement_tracking_path+ end++ def self.track_session_engagement(env)+ payload = read_json_body(env)+ return unless payload.is_a?(Hash)++ Scheduler::Defer.later("Track session engagement") do+ next if Discourse.pg_readonly_mode?++ BrowserPageviewSessionEngagement.upsert_from_payload(+ session_id: payload["session_id"],+ mouse_move_events: payload["mouse_move_events"].to_i,+ click_events: payload["click_events"].to_i,+ key_events: payload["key_events"].to_i,+ scroll_events: payload["scroll_events"].to_i,+ touch_events: payload["touch_events"].to_i,+ back_forward_events: payload["back_forward_events"].to_i,+ engaged_seconds:+ payload["engaged_seconds"].to_i.clamp(+ 0,+ SiteSetting.browser_pageview_max_engaged_seconds,+ ),+ time_to_first_interaction_ms: payload["time_to_first_interaction_ms"].presence&.to_i,+ )+ rescue => e+ Rails.logger.warn("Discarding session engagement: #{e.message}")+ end+ end++ def self.read_json_body(env)body = env["rack.input"]&.readenv["rack.input"]&.rewind- data =- begin- JSON.parse(body)- rescue JSON::ParserError- {}- end+ return if body.blank?++ JSON.parse(body)+ rescue JSON::ParserError+ nil+ end+ private_class_method :read_json_body++ def self.extract_beacon_view_tracking_data(env)+ data = read_json_body(env) || {}topic_id = data["topic_id"]&.to_itracking_url = data["url"]&.slice(0, MAX_URL_LENGTH)diff --git a/spec/helpers/application_helper_spec.rb b/spec/helpers/application_helper_spec.rbindex 7e91f9b7535..5e27c5a7199 100644--- a/spec/helpers/application_helper_spec.rb+++ b/spec/helpers/application_helper_spec.rb@@ -32,6 +32,26 @@ RSpec.describe ApplicationHelper doexpect(tags).to include('name="discourse-track-view-session-id"')expect(tags).to include('name="discourse-beacon-pageview-enabled"')end++ it "includes the engagement tracking meta tag when pageview events are persisted" do+ SiteSetting.dashboard_improvements = true+ SiteSetting.persist_browser_pageview_events = true++ tags = helper.discourse_pageview_tracking_meta_tags++ expect(tags).to include('name="discourse-engagement-tracking-enabled"')+ end++ it "omits the engagement tracking meta tag in a trigger-only config so the browser does not send /srv/se beacons the server rejects" do+ SiteSetting.dashboard_improvements = true+ SiteSetting.persist_browser_pageview_events = false+ SiteSetting.trigger_browser_pageview_events = true++ tags = helper.discourse_pageview_tracking_meta_tags++ expect(tags).to include('name="discourse-beacon-pageview-enabled"')+ expect(tags).not_to include('name="discourse-engagement-tracking-enabled"')+ endenddescribe "preload_script" dodiff --git a/spec/jobs/clean_up_browser_pageview_events_spec.rb b/spec/jobs/clean_up_browser_pageview_events_spec.rbindex 1242f870a41..f2b13a6efef 100644--- a/spec/jobs/clean_up_browser_pageview_events_spec.rb+++ b/spec/jobs/clean_up_browser_pageview_events_spec.rb@@ -18,6 +18,17 @@ RSpec.describe Jobs::CleanUpBrowserPageviewEvents doexpect(BrowserPageviewEvent.all).to contain_exactly(fresh_event)end+ it "deletes session engagements older than 3 months and keeps fresher ones" do+ SiteSetting.clean_up_browser_pageview_events = true+ fresh = BrowserPageviewSessionEngagement.create!(session_id: "fresh", created_at: 1.month.ago)+ BrowserPageviewSessionEngagement.create!(session_id: "old", created_at: 4.months.ago)++ expect { described_class.new.execute({}) }.to change {+ BrowserPageviewSessionEngagement.count+ }.by(-1)+ expect(BrowserPageviewSessionEngagement.all).to contain_exactly(fresh)+ end+it "deletes scores for deleted events" doSiteSetting.clean_up_browser_pageview_events = truefresh_event = Fabricate(:browser_pageview_event, created_at: 1.month.ago)diff --git a/spec/lib/middleware/request_tracker_spec.rb b/spec/lib/middleware/request_tracker_spec.rbindex 5858bc3cb1c..904d6499095 100644--- a/spec/lib/middleware/request_tracker_spec.rb+++ b/spec/lib/middleware/request_tracker_spec.rb@@ -1959,4 +1959,172 @@ RSpec.describe Middleware::RequestTracker doexpect(fake_logger.warnings).to be_emptyendend++ describe "session engagement tracking via /srv/se" do+ before do+ SiteSetting.dashboard_improvements = true+ SiteSetting.persist_browser_pageview_events = true+ end++ def engagement_env(body_hash, extra = {})+ env(+ {+ :path => "/srv/se",+ "HTTP_HOST" => "test.localhost",+ "REQUEST_METHOD" => "POST",+ "CONTENT_TYPE" => "application/json",+ "rack.input" => StringIO.new(JSON.generate(body_hash)),+ }.merge(extra),+ )+ end++ let(:same_origin) { { "HTTP_ORIGIN" => "http://test.localhost" } }+ let(:payload) do+ {+ session_id: "sess-1",+ mouse_move_events: 12,+ click_events: 3,+ key_events: 5,+ scroll_events: 7,+ touch_events: 0,+ back_forward_events: 1,+ engaged_seconds: 420,+ time_to_first_interaction_ms: 800,+ }+ end++ it "returns 204 and inserts the row for a same-origin request" do+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })++ expect {+ status, = middleware.call(engagement_env(payload, same_origin))+ expect(status).to eq(204)+ }.to change { BrowserPageviewSessionEngagement.count }.by(1)++ row = BrowserPageviewSessionEngagement.find_by(session_id: "sess-1")+ expect(row.mouse_move_events).to eq(12)+ expect(row.back_forward_events).to eq(1)+ expect(row.engaged_seconds).to eq(420)+ expect(row.time_to_first_interaction_ms).to eq(800)+ end++ it "updates the existing row on a later snapshot for the same session" do+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })+ middleware.call(engagement_env(payload, same_origin))++ later = payload.merge(mouse_move_events: 40, engaged_seconds: 900)++ expect { middleware.call(engagement_env(later, same_origin)) }.not_to change {+ BrowserPageviewSessionEngagement.count+ }++ row = BrowserPageviewSessionEngagement.find_by(session_id: "sess-1")+ expect(row.mouse_move_events).to eq(40)+ expect(row.engaged_seconds).to eq(900)+ end++ it "returns 403 and writes nothing for a cross-origin request" do+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })++ expect {+ status, = middleware.call(engagement_env(payload, "HTTP_ORIGIN" => "https://evil.example"))+ expect(status).to eq(403)+ }.not_to change { BrowserPageviewSessionEngagement.count }+ end++ it "handles a malformed JSON body without writing a row" do+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })++ expect {+ status, =+ middleware.call(engagement_env({}, same_origin).merge("rack.input" => StringIO.new("x")))+ expect(status).to eq(204)+ }.not_to change { BrowserPageviewSessionEngagement.count }+ end++ it "ignores a parseable JSON body that is not an object" do+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })++ expect {+ status, =+ middleware.call(+ engagement_env({}, same_origin).merge("rack.input" => StringIO.new("[1,2,3]")),+ )+ expect(status).to eq(204)+ }.not_to change { BrowserPageviewSessionEngagement.count }+ end++ it "coerces string and float metric values to integers" do+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })++ middleware.call(+ engagement_env(payload.merge(click_events: "9", engaged_seconds: 1234.9), same_origin),+ )++ expect(BrowserPageviewSessionEngagement.find_by(session_id: "sess-1")).to have_attributes(+ click_events: 9,+ engaged_seconds: 1234,+ )+ end++ it "treats a null or blank time to first interaction as null rather than zero" do+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })++ [nil, ""].each do |value|+ middleware.call(+ engagement_env(payload.merge(time_to_first_interaction_ms: value), same_origin),+ )++ expect(+ BrowserPageviewSessionEngagement.find_by(+ session_id: "sess-1",+ ).time_to_first_interaction_ms,+ ).to be_nil+ end+ end++ it "discards malformed-but-parseable payloads instead of raising in the deferred write" do+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })++ [+ payload.merge(session_id: 123),+ payload.merge(mouse_move_events: { "x" => 1 }),+ payload.merge(mouse_move_events: 9_999_999_999),+ ].each do |malformed|+ expect {+ status, = middleware.call(engagement_env(malformed, same_origin))+ expect(status).to eq(204)+ }.not_to change { BrowserPageviewSessionEngagement.count }+ end+ end++ it "clamps engaged seconds to the configured maximum" do+ SiteSetting.browser_pageview_max_engaged_seconds = 60+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })++ middleware.call(engagement_env(payload.merge(engaged_seconds: 5000), same_origin))++ expect(BrowserPageviewSessionEngagement.find_by(session_id: "sess-1").engaged_seconds).to eq(+ 60,+ )+ end++ it "writes nothing when the database is in readonly mode" do+ Discourse.stubs(:pg_readonly_mode?).returns(true)+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })++ expect {+ status, = middleware.call(engagement_env(payload, same_origin))+ expect(status).to eq(204)+ }.not_to change { BrowserPageviewSessionEngagement.count }+ end++ it "does not intercept the request when dashboard_improvements is disabled" do+ SiteSetting.dashboard_improvements = false+ middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })++ status, = middleware.call(engagement_env(payload, same_origin))+ expect(status).to eq(200)+ end+ endenddiff --git a/spec/models/browser_pageview_session_engagement_spec.rb b/spec/models/browser_pageview_session_engagement_spec.rbnew file mode 100644index 00000000000..f52a7f1d210--- /dev/null+++ b/spec/models/browser_pageview_session_engagement_spec.rb@@ -0,0 +1,87 @@+# frozen_string_literal: true++RSpec.describe BrowserPageviewSessionEngagement do+ describe ".upsert_from_payload" do+ let(:attributes) do+ {+ session_id: "sess-1",+ mouse_move_events: 12,+ click_events: 3,+ key_events: 5,+ scroll_events: 7,+ touch_events: 0,+ back_forward_events: 1,+ engaged_seconds: 4200,+ time_to_first_interaction_ms: 800,+ }+ end++ it "inserts a row with the given metrics" do+ expect { described_class.upsert_from_payload(**attributes) }.to change {+ described_class.count+ }.by(1)++ expect(described_class.find_by(session_id: "sess-1")).to have_attributes(+ mouse_move_events: 12,+ click_events: 3,+ key_events: 5,+ scroll_events: 7,+ touch_events: 0,+ back_forward_events: 1,+ engaged_seconds: 4200,+ time_to_first_interaction_ms: 800,+ )+ end++ it "stores a null time to first interaction when none is reported" do+ described_class.upsert_from_payload(**attributes.merge(time_to_first_interaction_ms: nil))++ expect(described_class.find_by(session_id: "sess-1").time_to_first_interaction_ms).to be_nil+ end++ it "truncates an over-long session id to the column limit" do+ long_id = "a" * (described_class::MAX_SESSION_ID_LENGTH + 10)++ described_class.upsert_from_payload(**attributes.merge(session_id: long_id))++ expect(described_class.first.session_id).to eq("a" * described_class::MAX_SESSION_ID_LENGTH)+ end++ it "writes nothing when the session id is blank" do+ expect {+ described_class.upsert_from_payload(**attributes.merge(session_id: ""))+ described_class.upsert_from_payload(**attributes.merge(session_id: nil))+ }.not_to change { described_class.count }+ end++ it "updates the existing row for the same session instead of duplicating" do+ described_class.upsert_from_payload(**attributes)++ expect {+ described_class.upsert_from_payload(+ **attributes.merge(mouse_move_events: 40, engaged_seconds: 9000),+ )+ }.not_to change { described_class.count }++ expect(described_class.find_by(session_id: "sess-1")).to have_attributes(+ mouse_move_events: 40,+ engaged_seconds: 9000,+ )+ end++ it "keeps the largest value per column across snapshots" do+ described_class.upsert_from_payload(+ **attributes.merge(mouse_move_events: 40, click_events: 3),+ )++ described_class.upsert_from_payload(+ **attributes.merge(mouse_move_events: 12, click_events: 20),+ )++ expect(described_class.find_by(session_id: "sess-1")).to have_attributes(+ mouse_move_events: 40,+ click_events: 20,+ )+ end+ end+enddiff --git a/spec/system/detect_human_activity_spec.rb b/spec/system/detect_human_activity_spec.rbnew file mode 100644index 00000000000..ebc1842c237--- /dev/null+++ b/spec/system/detect_human_activity_spec.rb@@ -0,0 +1,85 @@+# frozen_string_literal: true++describe "Detect human activity" do+ fab!(:topics) { Fabricate.times(5, :post).map(&:topic) }+ let(:discovery) { PageObjects::Pages::Discovery.new }++ before do+ SiteSetting.dashboard_improvements = true+ SiteSetting.persist_browser_pageview_events = true+ end++ def flush_engagement+ page.driver.with_playwright_page do |pw_page|+ pw_page.evaluate("window.dispatchEvent(new Event('pagehide'))")+ end+ end++ def engagement+ BrowserPageviewSessionEngagement.first+ end++ it "stores continuous mouse movement and time to first interaction" do+ visit("/")+ expect(discovery.topic_list).to have_topics(count: 5)++ # `steps` interpolates the movement into many small mousemove events,+ # mimicking the continuous path a real pointer traces.+ page.driver.with_playwright_page { |pw_page| pw_page.mouse.move(400, 400, steps: 30) }+ flush_engagement++ try_until_success { expect(BrowserPageviewSessionEngagement.count).to eq(1) }+ expect(engagement.mouse_move_events).to be > 0+ expect(engagement.time_to_first_interaction_ms).to be_present+ end++ it "ignores teleporting mouse jumps" do+ visit("/")+ expect(discovery.topic_list).to have_topics(count: 5)++ # Each move jumps straight to its destination with no intermediate events,+ # the way a script driving the cursor would. A real keypress is added so a+ # snapshot is actually sent (teleports alone leave the session empty).+ page.driver.with_playwright_page do |pw_page|+ pw_page.mouse.move(50, 50)+ pw_page.mouse.move(600, 600)+ pw_page.mouse.move(50, 600)+ pw_page.keyboard.press("a")+ end+ flush_engagement++ try_until_success { expect(BrowserPageviewSessionEngagement.count).to eq(1) }+ expect(engagement.mouse_move_events).to eq(0)+ expect(engagement.key_events).to eq(1)+ end++ it "accumulates engaged duration while the tab is visible and focused" do+ visit("/")+ expect(discovery.topic_list).to have_topics(count: 5)++ # An interaction is needed so a snapshot is sent; then stay engaged for a+ # beat before flushing so the duration has time to accumulate.+ page.driver.with_playwright_page do |pw_page|+ pw_page.keyboard.press("a")+ pw_page.wait_for_timeout(1200)+ end+ flush_engagement++ try_until_success { expect(BrowserPageviewSessionEngagement.count).to eq(1) }+ expect(engagement.engaged_seconds).to be >= 1+ end++ it "counts back/forward navigation" do+ visit("/")+ expect(discovery.topic_list).to have_topics(count: 5)++ # In-app navigation uses pushState (no popstate); the back button does fire+ # popstate, which is what we count.+ discovery.topic_list.visit_topic(topics[0])+ page.go_back+ flush_engagement++ try_until_success { expect(BrowserPageviewSessionEngagement.count).to eq(1) }+ expect(engagement.back_forward_events).to eq(1)+ end+end