Skip to content

A platform for community discussion. Free, open, simple.

PERF: Reduce session engagement beacon traffic (#42712)

Krzysztof Kotlarek7d240d5331f6
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
index 0dcc76018a0..ead34b7befb 100644
--- a/app/helpers/application_helper.rb
+++ b/app/helpers/application_helper.rb
@@ -446,6 +446,7 @@ module ApplicationHelper
if !SiteSetting.trigger_browser_pageview_events && !SiteSetting.persist_browser_pageview_events
return ""
end
+ return "" if Rails.env.development? && ENV["TRACK_REQUESTS"].blank?
tags = +""
tags << tag.meta(
diff --git a/frontend/discourse/app/services/human-activity-tracker.js b/frontend/discourse/app/services/human-activity-tracker.js
index 1ee90f28f0e..c7025a6b442 100644
--- a/frontend/discourse/app/services/human-activity-tracker.js
+++ b/frontend/discourse/app/services/human-activity-tracker.js
@@ -21,9 +21,10 @@ const EVENT_COUNTERS = {
const MAX_HUMAN_STEP = 100;
-const FLUSH_DELAY_MS = 3 * 60 * 1000;
+const FLUSH_DELAY_MS = 10 * 60 * 1000;
const THROTTLE_MS = 3000;
+const ENGAGEMENT_THRESHOLD_SECONDS = 10;
export default class HumanActivityTracker extends Service {
@service siteSettings;
@@ -49,6 +50,8 @@ export default class HumanActivityTracker extends Service {
#engagedMs = 0;
#engagedSince = null;
#lastSentMs = null;
+ #sentEngagementThreshold = false;
+ #sentEventCategories = new Set();
#counts;
#activityListener;
#mouseMoveListener;
@@ -86,7 +89,7 @@ export default class HumanActivityTracker extends Service {
this.#mouseMoveListener = (event) => this.#handleMouseMove(event);
this.#attentionListener = (attention) =>
this.#handleAttentionChange(attention);
- this.#pagehideListener = () => this.#flush({ force: true });
+ this.#pagehideListener = () => this.#flush({ final: true });
Object.keys(EVENT_COUNTERS).forEach((eventName) => {
window.addEventListener(eventName, this.#activityListener, {
@@ -204,26 +207,47 @@ export default class HumanActivityTracker extends Service {
};
}
- #flush({ force = false } = {}) {
- if (force && this.#flushTimer) {
+ #eventCategories() {
+ return Object.entries(this.#counts)
+ .filter(([, count]) => count > 0)
+ .map(([category]) => category);
+ }
+
+ #flush({ final = false } = {}) {
+ if (final && this.#flushTimer) {
cancel(this.#flushTimer);
this.#flushTimer = null;
}
+ const eventCategories = this.#eventCategories();
+ if (eventCategories.length < 2) {
+ return;
+ }
- const total = Object.values(this.#counts).reduce((sum, n) => sum + n, 0);
- if (total === 0) {
+ const hasNewEventCategory = eventCategories.some(
+ (category) => !this.#sentEventCategories.has(category)
+ );
+ const reachedEngagementThresholdForFirstTime =
+ !this.#sentEngagementThreshold &&
+ this.#engagedSeconds() >= ENGAGEMENT_THRESHOLD_SECONDS;
+ if (
+ !final &&
+ !hasNewEventCategory &&
+ !reachedEngagementThresholdForFirstTime
+ ) {
return;
}
const now = this.now();
if (
- !force &&
+ !final &&
this.#lastSentMs !== null &&
now - this.#lastSentMs < THROTTLE_MS
) {
return;
}
this.#lastSentMs = now;
+ this.#sentEngagementThreshold ||= reachedEngagementThresholdForFirstTime;
+ this.#sentEventCategories = new Set(eventCategories);
this.transport(this.#buildPayload());
}
diff --git a/frontend/discourse/scripts/js/pageview.js b/frontend/discourse/scripts/js/pageview.js
index 16515d24374..5b7550f046c 100644
--- a/frontend/discourse/scripts/js/pageview.js
+++ b/frontend/discourse/scripts/js/pageview.js
@@ -10,6 +10,9 @@ document.addEventListener("DOMContentLoaded", function () {
const trackViewSessionId = document.querySelector(
"meta[name=discourse-track-view-session-id]"
)?.content;
+ if (!trackViewSessionId) {
+ return;
+ }
const useBeacon =
document.querySelector("meta[name=discourse-beacon-pageview-enabled]")
diff --git a/frontend/discourse/tests/unit/services/human-activity-tracker-test.js b/frontend/discourse/tests/unit/services/human-activity-tracker-test.js
index 36d02600bc6..a1b34852601 100644
--- a/frontend/discourse/tests/unit/services/human-activity-tracker-test.js
+++ b/frontend/discourse/tests/unit/services/human-activity-tracker-test.js
@@ -68,6 +68,47 @@ module("Unit | Service | human-activity-tracker", function (hooks) {
assert.strictEqual(this.sent.length, 0);
});
+ test("waits for a scheduled flush after two interaction categories are recorded", function (assert) {
+ this.clock.ms = 6000;
+ window.dispatchEvent(new Event("keydown"));
+
+ assert.strictEqual(this.sent.length, 0);
+
+ window.dispatchEvent(new Event("mousedown"));
+
+ assert.strictEqual(this.sent.length, 0);
+
+ this.clock.ms = 600_000;
+ this.flushTick();
+
+ assert.strictEqual(this.sent.length, 1);
+ });
+
+ test("sends one update after engagement reaches ten seconds", async function (assert) {
+ await triggerEvent(document.body, "keydown");
+ await triggerEvent(document.body, "mousedown");
+
+ this.clock.ms = 5000;
+ blur(this);
+
+ this.clock.ms = 6000;
+ focus(this);
+
+ this.clock.ms = 11_000;
+ blur(this);
+
+ assert.strictEqual(this.sent.length, 2);
+ assert.strictEqual(this.sent.at(-1).engaged_seconds, 10);
+
+ this.clock.ms = 12_000;
+ focus(this);
+
+ this.clock.ms = 17_000;
+ blur(this);
+
+ assert.strictEqual(this.sent.length, 2);
+ });
+
test("ignores untrusted synthetic events", async function (assert) {
this.tracker.trustedEvent = (event) => event.isTrusted;
@@ -114,6 +155,7 @@ module("Unit | Service | human-activity-tracker", function (hooks) {
clientX: 900,
clientY: 900,
});
+ await triggerEvent(document.body, "keydown");
pagehide();
assert.strictEqual(this.sent.at(-1).mouse_move_events, 1);
@@ -122,6 +164,7 @@ module("Unit | Service | human-activity-tracker", function (hooks) {
test("reports the time to the first interaction", async function (assert) {
this.clock.ms = 2500;
await triggerEvent(document.body, "keydown");
+ await triggerEvent(document.body, "mousedown");
this.clock.ms = 9000;
pagehide();
@@ -131,6 +174,7 @@ module("Unit | Service | human-activity-tracker", function (hooks) {
test("accumulates only visible-and-focused time as engaged duration", async function (assert) {
await triggerEvent(document.body, "keydown");
+ await triggerEvent(document.body, "mousedown");
this.clock.ms = 4000;
blur(this);
@@ -146,6 +190,7 @@ module("Unit | Service | human-activity-tracker", function (hooks) {
test("flushes the latest snapshot when the tab is hidden", async function (assert) {
await triggerEvent(document.body, "keydown");
+ await triggerEvent(document.body, "mousedown");
this.clock.ms = 5000;
hide(this);
@@ -157,6 +202,7 @@ module("Unit | Service | human-activity-tracker", function (hooks) {
test("caps engaged seconds at the configured maximum", async function (assert) {
this.tracker.siteSettings.browser_pageview_max_engaged_seconds = 5;
await triggerEvent(document.body, "keydown");
+ await triggerEvent(document.body, "mousedown");
this.clock.ms = 9000;
pagehide();
@@ -166,48 +212,67 @@ module("Unit | Service | human-activity-tracker", function (hooks) {
test("throttles sends to at most one every three seconds", async function (assert) {
await triggerEvent(document.body, "keydown");
+ await triggerEvent(document.body, "mousedown");
this.clock.ms = 1000;
- blur(this);
+ this.flushTick();
this.clock.ms = 2000;
- focus(this);
- this.clock.ms = 3000;
- blur(this);
+ await triggerEvent(document.body, "scroll");
+ this.flushTick();
assert.strictEqual(this.sent.length, 1);
this.clock.ms = 6000;
- focus(this);
- this.clock.ms = 7000;
- blur(this);
+ await triggerEvent(document.body, "touchstart");
+ this.flushTick();
assert.strictEqual(this.sent.length, 2);
});
test("always flushes on pagehide, bypassing the throttle", async function (assert) {
await triggerEvent(document.body, "keydown");
+ await triggerEvent(document.body, "mousedown");
this.clock.ms = 1000;
- blur(this);
+ this.flushTick();
this.clock.ms = 2000;
- focus(this);
- this.clock.ms = 3000;
+ await triggerEvent(document.body, "scroll");
+ this.clock.ms = 2500;
pagehide();
assert.strictEqual(this.sent.length, 2);
});
- test("keeps sending periodic snapshots on each flush cadence", async function (assert) {
+ test("sends periodic snapshots only after a new event category is recorded", async function (assert) {
await triggerEvent(document.body, "keydown");
+ await triggerEvent(document.body, "mousedown");
+
+ this.clock.ms = 600_000;
+ this.flushTick();
+
+ this.clock.ms = 1_200_000;
+ this.flushTick();
+
+ await triggerEvent(document.body, "scroll");
- this.clock.ms = 180_000;
+ this.clock.ms = 1_800_000;
this.flushTick();
- this.clock.ms = 360_000;
+ assert.strictEqual(this.sent.length, 2);
+ });
+
+ test("sends a final snapshot on pagehide even without a new event category", async function (assert) {
+ await triggerEvent(document.body, "keydown");
+ await triggerEvent(document.body, "mousedown");
+
+ this.clock.ms = 600_000;
this.flushTick();
+ this.clock.ms = 1_200_000;
+ pagehide();
+
assert.strictEqual(this.sent.length, 2);
});
});
diff --git a/spec/helpers/application_helper_spec.rb b/spec/helpers/application_helper_spec.rb
index 8fb4838e7a5..7bed094ac3e 100644
--- a/spec/helpers/application_helper_spec.rb
+++ b/spec/helpers/application_helper_spec.rb
@@ -22,6 +22,17 @@ RSpec.describe ApplicationHelper do
expect(tags).not_to include('name="discourse-beacon-pageview-enabled"')
end
+ it "omits tracking meta tags in development when request tracking is not enabled" do
+ original_track_requests = ENV.delete("TRACK_REQUESTS")
+ Rails.env.stubs(:development?).returns(true)
+
+ tags = helper.discourse_pageview_tracking_meta_tags
+
+ expect(tags).to be_blank
+ ensure
+ ENV["TRACK_REQUESTS"] = original_track_requests if original_track_requests
+ end
+
it "includes beacon tracking meta tags for browser pageview event triggers" do
SiteSetting.dashboard_improvements = true
SiteSetting.persist_browser_pageview_events = false
diff --git a/spec/system/detect_human_activity_spec.rb b/spec/system/detect_human_activity_spec.rb
index ebc1842c237..09551a2d1b9 100644
--- a/spec/system/detect_human_activity_spec.rb
+++ b/spec/system/detect_human_activity_spec.rb
@@ -26,6 +26,7 @@ describe "Detect human activity" do
# `steps` interpolates the movement into many small mousemove events,
# mimicking the continuous path a real pointer traces.
page.driver.with_playwright_page { |pw_page| pw_page.mouse.move(400, 400, steps: 30) }
+ page.driver.with_playwright_page { |pw_page| pw_page.keyboard.press("a") }
flush_engagement
try_until_success { expect(BrowserPageviewSessionEngagement.count).to eq(1) }
@@ -38,13 +39,15 @@ describe "Detect human activity" do
expect(discovery.topic_list).to have_topics(count: 5)
# Each move jumps straight to its destination with no intermediate events,
- # the way a script driving the cursor would. A real keypress is added so a
- # snapshot is actually sent (teleports alone leave the session empty).
+ # the way a script driving the cursor would. A real keypress and click meet
+ # the two-category threshold so a snapshot is sent.
page.driver.with_playwright_page do |pw_page|
pw_page.mouse.move(50, 50)
pw_page.mouse.move(600, 600)
pw_page.mouse.move(50, 600)
pw_page.keyboard.press("a")
+ pw_page.mouse.down
+ pw_page.mouse.up
end
flush_engagement
@@ -61,6 +64,8 @@ describe "Detect human activity" do
# beat before flushing so the duration has time to accumulate.
page.driver.with_playwright_page do |pw_page|
pw_page.keyboard.press("a")
+ pw_page.mouse.down
+ pw_page.mouse.up
pw_page.wait_for_timeout(1200)
end
flush_engagement
@@ -77,6 +82,7 @@ describe "Detect human activity" do
# popstate, which is what we count.
discovery.topic_list.visit_topic(topics[0])
page.go_back
+ page.driver.with_playwright_page { |pw_page| pw_page.keyboard.press("a") }
flush_engagement
try_until_success { expect(BrowserPageviewSessionEngagement.count).to eq(1) }
diff --git a/spec/system/standalone_scripts_spec.rb b/spec/system/standalone_scripts_spec.rb
index 506940b6ffa..22b117fe4db 100644
--- a/spec/system/standalone_scripts_spec.rb
+++ b/spec/system/standalone_scripts_spec.rb
@@ -111,6 +111,27 @@ describe "Standalone scripts" do
try_until_success { expect(pageview_requests).not_to be_empty }
end
+
+ it "does not send tracking requests when request tracking is disabled in development" do
+ original_track_requests = ENV.delete("TRACK_REQUESTS")
+ Rails.env.stubs(:development?).returns(true)
+ tracking_requests = []
+ page.driver.with_playwright_page do |pw_page|
+ pw_page.on(
+ "request",
+ lambda do |request|
+ tracking_requests << request.url if request.url.match?(%r{/(pageview|srv/pv)$})
+ end,
+ )
+ end
+
+ visit("/safe-mode")
+
+ page.driver.with_playwright_page { |pw_page| pw_page.wait_for_timeout(100) }
+ expect(tracking_requests).to eq([])
+ ensure
+ ENV["TRACK_REQUESTS"] = original_track_requests if original_track_requests
+ end
end
describe "publish.js" do