A platform for community discussion. Free, open, simple.
PERF: Reduce session engagement beacon traffic (#42712)
Krzysztof Kotlarek7d240d5331f6
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rbindex 0dcc76018a0..ead34b7befb 100644--- a/app/helpers/application_helper.rb+++ b/app/helpers/application_helper.rb@@ -446,6 +446,7 @@ module ApplicationHelperif !SiteSetting.trigger_browser_pageview_events && !SiteSetting.persist_browser_pageview_eventsreturn ""end+ return "" if Rails.env.development? && ENV["TRACK_REQUESTS"].blank?tags = +""tags << tag.meta(diff --git a/frontend/discourse/app/services/human-activity-tracker.js b/frontend/discourse/app/services/human-activity-tracker.jsindex 1ee90f28f0e..c7025a6b442 100644--- a/frontend/discourse/app/services/human-activity-tracker.js+++ b/frontend/discourse/app/services/human-activity-tracker.js@@ -21,9 +21,10 @@ const EVENT_COUNTERS = {const MAX_HUMAN_STEP = 100;-const FLUSH_DELAY_MS = 3 * 60 * 1000;+const FLUSH_DELAY_MS = 10 * 60 * 1000;const THROTTLE_MS = 3000;+const ENGAGEMENT_THRESHOLD_SECONDS = 10;export default class HumanActivityTracker extends Service {@service siteSettings;@@ -49,6 +50,8 @@ export default class HumanActivityTracker extends Service {#engagedMs = 0;#engagedSince = null;#lastSentMs = null;+ #sentEngagementThreshold = false;+ #sentEventCategories = new Set();#counts;#activityListener;#mouseMoveListener;@@ -86,7 +89,7 @@ export default class HumanActivityTracker extends Service {this.#mouseMoveListener = (event) => this.#handleMouseMove(event);this.#attentionListener = (attention) =>this.#handleAttentionChange(attention);- this.#pagehideListener = () => this.#flush({ force: true });+ this.#pagehideListener = () => this.#flush({ final: true });Object.keys(EVENT_COUNTERS).forEach((eventName) => {window.addEventListener(eventName, this.#activityListener, {@@ -204,26 +207,47 @@ export default class HumanActivityTracker extends Service {};}- #flush({ force = false } = {}) {- if (force && this.#flushTimer) {+ #eventCategories() {+ return Object.entries(this.#counts)+ .filter(([, count]) => count > 0)+ .map(([category]) => category);+ }++ #flush({ final = false } = {}) {+ if (final && this.#flushTimer) {cancel(this.#flushTimer);this.#flushTimer = null;}+ const eventCategories = this.#eventCategories();+ if (eventCategories.length < 2) {+ return;+ }- const total = Object.values(this.#counts).reduce((sum, n) => sum + n, 0);- if (total === 0) {+ const hasNewEventCategory = eventCategories.some(+ (category) => !this.#sentEventCategories.has(category)+ );+ const reachedEngagementThresholdForFirstTime =+ !this.#sentEngagementThreshold &&+ this.#engagedSeconds() >= ENGAGEMENT_THRESHOLD_SECONDS;+ if (+ !final &&+ !hasNewEventCategory &&+ !reachedEngagementThresholdForFirstTime+ ) {return;}const now = this.now();if (- !force &&+ !final &&this.#lastSentMs !== null &&now - this.#lastSentMs < THROTTLE_MS) {return;}this.#lastSentMs = now;+ this.#sentEngagementThreshold ||= reachedEngagementThresholdForFirstTime;+ this.#sentEventCategories = new Set(eventCategories);this.transport(this.#buildPayload());}diff --git a/frontend/discourse/scripts/js/pageview.js b/frontend/discourse/scripts/js/pageview.jsindex 16515d24374..5b7550f046c 100644--- a/frontend/discourse/scripts/js/pageview.js+++ b/frontend/discourse/scripts/js/pageview.js@@ -10,6 +10,9 @@ document.addEventListener("DOMContentLoaded", function () {const trackViewSessionId = document.querySelector("meta[name=discourse-track-view-session-id]")?.content;+ if (!trackViewSessionId) {+ return;+ }const useBeacon =document.querySelector("meta[name=discourse-beacon-pageview-enabled]")diff --git a/frontend/discourse/tests/unit/services/human-activity-tracker-test.js b/frontend/discourse/tests/unit/services/human-activity-tracker-test.jsindex 36d02600bc6..a1b34852601 100644--- a/frontend/discourse/tests/unit/services/human-activity-tracker-test.js+++ b/frontend/discourse/tests/unit/services/human-activity-tracker-test.js@@ -68,6 +68,47 @@ module("Unit | Service | human-activity-tracker", function (hooks) {assert.strictEqual(this.sent.length, 0);});+ test("waits for a scheduled flush after two interaction categories are recorded", function (assert) {+ this.clock.ms = 6000;+ window.dispatchEvent(new Event("keydown"));++ assert.strictEqual(this.sent.length, 0);++ window.dispatchEvent(new Event("mousedown"));++ assert.strictEqual(this.sent.length, 0);++ this.clock.ms = 600_000;+ this.flushTick();++ assert.strictEqual(this.sent.length, 1);+ });++ test("sends one update after engagement reaches ten seconds", async function (assert) {+ await triggerEvent(document.body, "keydown");+ await triggerEvent(document.body, "mousedown");++ this.clock.ms = 5000;+ blur(this);++ this.clock.ms = 6000;+ focus(this);++ this.clock.ms = 11_000;+ blur(this);++ assert.strictEqual(this.sent.length, 2);+ assert.strictEqual(this.sent.at(-1).engaged_seconds, 10);++ this.clock.ms = 12_000;+ focus(this);++ this.clock.ms = 17_000;+ blur(this);++ assert.strictEqual(this.sent.length, 2);+ });+test("ignores untrusted synthetic events", async function (assert) {this.tracker.trustedEvent = (event) => event.isTrusted;@@ -114,6 +155,7 @@ module("Unit | Service | human-activity-tracker", function (hooks) {clientX: 900,clientY: 900,});+ await triggerEvent(document.body, "keydown");pagehide();assert.strictEqual(this.sent.at(-1).mouse_move_events, 1);@@ -122,6 +164,7 @@ module("Unit | Service | human-activity-tracker", function (hooks) {test("reports the time to the first interaction", async function (assert) {this.clock.ms = 2500;await triggerEvent(document.body, "keydown");+ await triggerEvent(document.body, "mousedown");this.clock.ms = 9000;pagehide();@@ -131,6 +174,7 @@ module("Unit | Service | human-activity-tracker", function (hooks) {test("accumulates only visible-and-focused time as engaged duration", async function (assert) {await triggerEvent(document.body, "keydown");+ await triggerEvent(document.body, "mousedown");this.clock.ms = 4000;blur(this);@@ -146,6 +190,7 @@ module("Unit | Service | human-activity-tracker", function (hooks) {test("flushes the latest snapshot when the tab is hidden", async function (assert) {await triggerEvent(document.body, "keydown");+ await triggerEvent(document.body, "mousedown");this.clock.ms = 5000;hide(this);@@ -157,6 +202,7 @@ module("Unit | Service | human-activity-tracker", function (hooks) {test("caps engaged seconds at the configured maximum", async function (assert) {this.tracker.siteSettings.browser_pageview_max_engaged_seconds = 5;await triggerEvent(document.body, "keydown");+ await triggerEvent(document.body, "mousedown");this.clock.ms = 9000;pagehide();@@ -166,48 +212,67 @@ module("Unit | Service | human-activity-tracker", function (hooks) {test("throttles sends to at most one every three seconds", async function (assert) {await triggerEvent(document.body, "keydown");+ await triggerEvent(document.body, "mousedown");this.clock.ms = 1000;- blur(this);+ this.flushTick();this.clock.ms = 2000;- focus(this);- this.clock.ms = 3000;- blur(this);+ await triggerEvent(document.body, "scroll");+ this.flushTick();assert.strictEqual(this.sent.length, 1);this.clock.ms = 6000;- focus(this);- this.clock.ms = 7000;- blur(this);+ await triggerEvent(document.body, "touchstart");+ this.flushTick();assert.strictEqual(this.sent.length, 2);});test("always flushes on pagehide, bypassing the throttle", async function (assert) {await triggerEvent(document.body, "keydown");+ await triggerEvent(document.body, "mousedown");this.clock.ms = 1000;- blur(this);+ this.flushTick();this.clock.ms = 2000;- focus(this);- this.clock.ms = 3000;+ await triggerEvent(document.body, "scroll");+ this.clock.ms = 2500;pagehide();assert.strictEqual(this.sent.length, 2);});- test("keeps sending periodic snapshots on each flush cadence", async function (assert) {+ test("sends periodic snapshots only after a new event category is recorded", async function (assert) {await triggerEvent(document.body, "keydown");+ await triggerEvent(document.body, "mousedown");++ this.clock.ms = 600_000;+ this.flushTick();++ this.clock.ms = 1_200_000;+ this.flushTick();++ await triggerEvent(document.body, "scroll");- this.clock.ms = 180_000;+ this.clock.ms = 1_800_000;this.flushTick();- this.clock.ms = 360_000;+ assert.strictEqual(this.sent.length, 2);+ });++ test("sends a final snapshot on pagehide even without a new event category", async function (assert) {+ await triggerEvent(document.body, "keydown");+ await triggerEvent(document.body, "mousedown");++ this.clock.ms = 600_000;this.flushTick();+ this.clock.ms = 1_200_000;+ pagehide();+assert.strictEqual(this.sent.length, 2);});});diff --git a/spec/helpers/application_helper_spec.rb b/spec/helpers/application_helper_spec.rbindex 8fb4838e7a5..7bed094ac3e 100644--- a/spec/helpers/application_helper_spec.rb+++ b/spec/helpers/application_helper_spec.rb@@ -22,6 +22,17 @@ RSpec.describe ApplicationHelper doexpect(tags).not_to include('name="discourse-beacon-pageview-enabled"')end+ it "omits tracking meta tags in development when request tracking is not enabled" do+ original_track_requests = ENV.delete("TRACK_REQUESTS")+ Rails.env.stubs(:development?).returns(true)++ tags = helper.discourse_pageview_tracking_meta_tags++ expect(tags).to be_blank+ ensure+ ENV["TRACK_REQUESTS"] = original_track_requests if original_track_requests+ end+it "includes beacon tracking meta tags for browser pageview event triggers" doSiteSetting.dashboard_improvements = trueSiteSetting.persist_browser_pageview_events = falsediff --git a/spec/system/detect_human_activity_spec.rb b/spec/system/detect_human_activity_spec.rbindex ebc1842c237..09551a2d1b9 100644--- a/spec/system/detect_human_activity_spec.rb+++ b/spec/system/detect_human_activity_spec.rb@@ -26,6 +26,7 @@ describe "Detect human activity" do# `steps` interpolates the movement into many small mousemove events,# mimicking the continuous path a real pointer traces.page.driver.with_playwright_page { |pw_page| pw_page.mouse.move(400, 400, steps: 30) }+ page.driver.with_playwright_page { |pw_page| pw_page.keyboard.press("a") }flush_engagementtry_until_success { expect(BrowserPageviewSessionEngagement.count).to eq(1) }@@ -38,13 +39,15 @@ describe "Detect human activity" doexpect(discovery.topic_list).to have_topics(count: 5)# Each move jumps straight to its destination with no intermediate events,- # the way a script driving the cursor would. A real keypress is added so a- # snapshot is actually sent (teleports alone leave the session empty).+ # the way a script driving the cursor would. A real keypress and click meet+ # the two-category threshold so a snapshot is sent.page.driver.with_playwright_page do |pw_page|pw_page.mouse.move(50, 50)pw_page.mouse.move(600, 600)pw_page.mouse.move(50, 600)pw_page.keyboard.press("a")+ pw_page.mouse.down+ pw_page.mouse.upendflush_engagement@@ -61,6 +64,8 @@ describe "Detect human activity" do# beat before flushing so the duration has time to accumulate.page.driver.with_playwright_page do |pw_page|pw_page.keyboard.press("a")+ pw_page.mouse.down+ pw_page.mouse.uppw_page.wait_for_timeout(1200)endflush_engagement@@ -77,6 +82,7 @@ describe "Detect human activity" do# popstate, which is what we count.discovery.topic_list.visit_topic(topics[0])page.go_back+ page.driver.with_playwright_page { |pw_page| pw_page.keyboard.press("a") }flush_engagementtry_until_success { expect(BrowserPageviewSessionEngagement.count).to eq(1) }diff --git a/spec/system/standalone_scripts_spec.rb b/spec/system/standalone_scripts_spec.rbindex 506940b6ffa..22b117fe4db 100644--- a/spec/system/standalone_scripts_spec.rb+++ b/spec/system/standalone_scripts_spec.rb@@ -111,6 +111,27 @@ describe "Standalone scripts" dotry_until_success { expect(pageview_requests).not_to be_empty }end++ it "does not send tracking requests when request tracking is disabled in development" do+ original_track_requests = ENV.delete("TRACK_REQUESTS")+ Rails.env.stubs(:development?).returns(true)+ tracking_requests = []+ page.driver.with_playwright_page do |pw_page|+ pw_page.on(+ "request",+ lambda do |request|+ tracking_requests << request.url if request.url.match?(%r{/(pageview|srv/pv)$})+ end,+ )+ end++ visit("/safe-mode")++ page.driver.with_playwright_page { |pw_page| pw_page.wait_for_timeout(100) }+ expect(tracking_requests).to eq([])+ ensure+ ENV["TRACK_REQUESTS"] = original_track_requests if original_track_requests+ endenddescribe "publish.js" do