Skip to content

A platform for community discussion. Free, open, simple.

FIX: Split browser pageview rollup source at the cutover date (#42013)

Krzysztof Kotlarek820ebddd65ab
diff --git a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb
index 0147e824417..c7a4a68d67d 100644
--- a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb
+++ b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb
@@ -50,7 +50,7 @@ module Jobs
if BrowserPageviewCountryDailyRollup.none? && BrowserPageviewReferrerDailyRollup.none?
earliest_event_date =
BrowserPageviewEvent
- .where(source: BrowserPageviewEvent.rollup_source)
+ .where(BrowserPageviewEvent.rollup_source_condition)
.minimum(:created_at)
&.to_date
[earliest_event_date, end_date]
@@ -71,11 +71,7 @@ module Jobs
end
def next_batch
- params = {
- source: BrowserPageviewEvent.rollup_source,
- version: BrowserPageviewReferrerInspector::VERSION,
- limit: batch_size,
- }
+ params = { version: BrowserPageviewReferrerInspector::VERSION, limit: batch_size }
retention_clause = ""
if SiteSetting.clean_up_browser_pageview_events
@@ -90,7 +86,7 @@ module Jobs
SELECT id, referrer
FROM browser_pageview_events
WHERE referrer IS NOT NULL
- AND source = :source
+ AND #{BrowserPageviewEvent.rollup_source_condition}
AND (
normalized_referrer_version IS NULL
OR normalized_referrer_version < :version
@@ -117,11 +113,7 @@ module Jobs
end
def recomputable_dates(ids)
- params = {
- ids: ids,
- source: BrowserPageviewEvent.rollup_source,
- version: BrowserPageviewReferrerInspector::VERSION,
- }
+ params = { ids: ids, version: BrowserPageviewReferrerInspector::VERSION }
retention_clause = ""
if SiteSetting.clean_up_browser_pageview_events
@@ -145,7 +137,7 @@ module Jobs
FROM browser_pageview_events e
WHERE e.created_at >= touched_dates.date
AND e.created_at < touched_dates.date + 1
- AND e.source = :source
+ AND #{BrowserPageviewEvent.rollup_source_condition(table: "e")}
AND e.referrer IS NOT NULL
AND NOT EXISTS (
SELECT 1
diff --git a/app/models/browser_pageview_country_daily_rollup.rb b/app/models/browser_pageview_country_daily_rollup.rb
index eca37ce1289..d7f3e7a29cb 100644
--- a/app/models/browser_pageview_country_daily_rollup.rb
+++ b/app/models/browser_pageview_country_daily_rollup.rb
@@ -1,11 +1,11 @@
# frozen_string_literal: true
class BrowserPageviewCountryDailyRollup < ActiveRecord::Base
- def self.aggregate(start_date:, end_date:, source: BrowserPageviewEvent.rollup_source)
+ def self.aggregate(start_date:, end_date:)
start_date = start_date.to_date
end_date = end_date.to_date + 1
- DB.exec(<<~SQL, start_date:, end_date:, source:)
+ DB.exec(<<~SQL, start_date:, end_date:)
INSERT INTO browser_pageview_country_daily_rollups (date, country_code, count, logged_in_count)
SELECT
created_at::date AS date,
@@ -15,7 +15,7 @@ class BrowserPageviewCountryDailyRollup < ActiveRecord::Base
FROM browser_pageview_events
WHERE created_at >= :start_date
AND created_at < :end_date
- AND source = :source
+ AND #{BrowserPageviewEvent.rollup_source_condition}
GROUP BY date, country_code
ON CONFLICT (date, country_code) DO UPDATE
SET count = EXCLUDED.count,
diff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rb
index 089fac915f6..7b813bfaf28 100644
--- a/app/models/browser_pageview_event.rb
+++ b/app/models/browser_pageview_event.rb
@@ -212,12 +212,19 @@ class BrowserPageviewEvent < ActiveRecord::Base
RETENTION_PERIOD.ago.beginning_of_day
end
- def self.rollup_source
- if UpcomingChanges.enabled?(:dashboard_improvements)
- SOURCE_BEACON
- else
- SOURCE_PIGGYBACK
- end
+ def self.rollup_source_condition(table: nil)
+ prefix = table ? "#{table}." : ""
+ cutover_date = beacon_cutover_date
+ return "#{prefix}source = #{SOURCE_PIGGYBACK}" if cutover_date.nil?
+
+ sanitize_sql_array(
+ [
+ "(#{prefix}created_at < ? AND #{prefix}source = #{SOURCE_PIGGYBACK} " \
+ "OR #{prefix}created_at >= ? AND #{prefix}source = #{SOURCE_BEACON})",
+ cutover_date,
+ cutover_date,
+ ],
+ )
end
before_save :truncate_fields
diff --git a/app/models/browser_pageview_referrer_daily_rollup.rb b/app/models/browser_pageview_referrer_daily_rollup.rb
index 1d24649510c..5c8d1e3c51f 100644
--- a/app/models/browser_pageview_referrer_daily_rollup.rb
+++ b/app/models/browser_pageview_referrer_daily_rollup.rb
@@ -1,11 +1,11 @@
# frozen_string_literal: true
class BrowserPageviewReferrerDailyRollup < ActiveRecord::Base
- def self.aggregate(start_date:, end_date:, source: BrowserPageviewEvent.rollup_source)
+ def self.aggregate(start_date:, end_date:)
start_date = start_date.to_date
end_date = end_date.to_date + 1
- DB.exec(<<~SQL, start_date:, end_date:, source:)
+ DB.exec(<<~SQL, start_date:, end_date:)
INSERT INTO browser_pageview_referrer_daily_rollups (date, normalized_referrer, count, logged_in_count)
SELECT
created_at::date AS date,
@@ -15,7 +15,7 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::Base
FROM browser_pageview_events
WHERE created_at >= :start_date
AND created_at < :end_date
- AND source = :source
+ AND #{BrowserPageviewEvent.rollup_source_condition}
GROUP BY date, normalized_referrer
ON CONFLICT (date, normalized_referrer) DO UPDATE
SET count = EXCLUDED.count,
@@ -27,13 +27,11 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::Base
dates = Array(dates).map(&:to_date).uniq
return if dates.empty?
- source = BrowserPageviewEvent.rollup_source
-
# The rollups are the permanent record, but their source events are pruned
# after a retention period (CleanUpBrowserPageviewEvents). Only rebuild
# dates that still have events so we never delete a rollup we can no longer
# reconstruct from events.
- dates = DB.query_single(<<~SQL, dates:, source:)
+ dates = DB.query_single(<<~SQL, dates:)
SELECT d.date
FROM unnest(ARRAY[:dates]::date[]) AS d(date)
WHERE EXISTS (
@@ -41,7 +39,7 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::Base
FROM browser_pageview_events e
WHERE e.created_at >= d.date
AND e.created_at < d.date + 1
- AND e.source = :source
+ AND #{BrowserPageviewEvent.rollup_source_condition(table: "e")}
)
SQL
return if dates.empty?
@@ -52,7 +50,7 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::Base
WHERE date IN (:dates)
SQL
- dates.each { |date| aggregate(start_date: date, end_date: date, source:) }
+ dates.each { |date| aggregate(start_date: date, end_date: date) }
end
end
end
diff --git a/app/models/browser_pageview_session_engagement_daily_rollup.rb b/app/models/browser_pageview_session_engagement_daily_rollup.rb
index 0890023cdd6..def526eb5eb 100644
--- a/app/models/browser_pageview_session_engagement_daily_rollup.rb
+++ b/app/models/browser_pageview_session_engagement_daily_rollup.rb
@@ -5,12 +5,13 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Base
MIN_SESSION_AGE = 10.minutes
private_constant :BOUNCE_ENGAGED_SECONDS_THRESHOLD, :MIN_SESSION_AGE
- def self.aggregate(start_date:, end_date:, source: BrowserPageviewEvent.rollup_source)
+ def self.aggregate(start_date:, end_date:)
start_date = start_date.to_date
end_date = end_date.to_date + 1
+ source_condition = BrowserPageviewEvent.rollup_source_condition
transaction do
- DB.exec(<<~SQL, start_date:, end_date:, source:)
+ DB.exec(<<~SQL, start_date:, end_date:)
DELETE FROM browser_pageview_session_engagement_daily_rollups rollup
WHERE rollup.date >= :start_date
AND rollup.date < :end_date
@@ -19,7 +20,7 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Base
FROM browser_pageview_events
WHERE created_at >= rollup.date
AND created_at < rollup.date + 1
- AND source = :source
+ AND #{source_condition}
)
SQL
@@ -30,7 +31,7 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Base
FROM browser_pageview_events
WHERE created_at >= :start_date
AND created_at < LEAST(:end_date::timestamp, :session_started_before::timestamp)
- AND source = :source
+ AND #{source_condition}
),
session_pageviews AS (
SELECT
@@ -40,7 +41,7 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Base
bool_or(bpe.user_id IS NOT NULL) AS logged_in
FROM browser_pageview_events bpe
JOIN active_sessions ON active_sessions.session_id = bpe.session_id
- WHERE bpe.source = :source
+ WHERE #{BrowserPageviewEvent.rollup_source_condition(table: "bpe")}
GROUP BY bpe.session_id
HAVING MIN(bpe.created_at) >= :start_date
)
@@ -64,7 +65,6 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Base
end_date:,
session_started_before: MIN_SESSION_AGE.ago,
bounce_threshold: BOUNCE_ENGAGED_SECONDS_THRESHOLD,
- source:,
)
end
end
diff --git a/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb b/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb
index 5ef71dffbe1..0426a0f6241 100644
--- a/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb
+++ b/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb
@@ -463,7 +463,7 @@ module DiscourseAi
AND e.created_at < (:end_date::date + 1)
AND e.topic_id IS NOT NULL
AND e.normalized_referrer IS NOT NULL
- AND e.source = :source
+ AND #{BrowserPageviewEvent.rollup_source_condition(table: "e")}
AND #{topic_conditions}
GROUP BY e.topic_id, t.title
ORDER BY visits DESC
@@ -471,7 +471,6 @@ module DiscourseAi
SQL
start_date: @start_date,
end_date: @end_date,
- source: BrowserPageviewEvent.rollup_source,
category_ids: scoped_category_ids,
).first
return if row.nil? || row.visits.to_i < 50
diff --git a/spec/jobs/maintain_browser_pageview_rollups_spec.rb b/spec/jobs/maintain_browser_pageview_rollups_spec.rb
index 4a6e332c912..b62f225cdaa 100644
--- a/spec/jobs/maintain_browser_pageview_rollups_spec.rb
+++ b/spec/jobs/maintain_browser_pageview_rollups_spec.rb
@@ -34,38 +34,53 @@ RSpec.describe Jobs::MaintainBrowserPageviewRollups do
).to eq(1)
end
- it "uses beacon source for rollups when dashboard_improvements is enabled" do
- SiteSetting.dashboard_improvements = false
+ it "rolls up piggyback events before the beacon cutover date and beacon events after it" do
+ freeze_time(Time.utc(2026, 6, 20, 12))
+ SiteSetting.dashboard_improvements = true
+ UpcomingChangeEvent.create!(
+ upcoming_change_name: "dashboard_improvements",
+ event_type: :manual_opt_in,
+ created_at: Time.utc(2026, 6, 10, 9),
+ )
+
Fabricate(
:browser_pageview_event,
country_code: "US",
normalized_referrer: "google.com",
- source: BrowserPageviewEvent::SOURCE_PIGGYBACK,
+ source: :piggyback,
+ created_at: Time.utc(2026, 6, 9, 10),
)
Fabricate(
:browser_pageview_event,
country_code: "GB",
normalized_referrer: "reddit.com",
- source: BrowserPageviewEvent::SOURCE_BEACON,
+ source: :beacon,
+ created_at: Time.utc(2026, 6, 9, 10),
)
-
- job.execute({})
-
- expect(BrowserPageviewCountryDailyRollup.pluck(:country_code, :count)).to eq([["US", 1]])
- expect(BrowserPageviewReferrerDailyRollup.pluck(:normalized_referrer, :count)).to eq(
- [["google.com", 1]],
+ Fabricate(
+ :browser_pageview_event,
+ country_code: "FR",
+ normalized_referrer: "bing.com",
+ source: :beacon,
+ created_at: Time.utc(2026, 6, 15, 10),
+ )
+ Fabricate(
+ :browser_pageview_event,
+ country_code: "DE",
+ normalized_referrer: "duckduckgo.com",
+ source: :piggyback,
+ created_at: Time.utc(2026, 6, 15, 10),
)
- SiteSetting.dashboard_improvements = true
job.execute({})
expect(BrowserPageviewCountryDailyRollup.pluck(:country_code, :count)).to contain_exactly(
["US", 1],
- ["GB", 1],
+ ["FR", 1],
)
expect(
BrowserPageviewReferrerDailyRollup.pluck(:normalized_referrer, :count),
- ).to contain_exactly(["google.com", 1], ["reddit.com", 1])
+ ).to contain_exactly(["google.com", 1], ["bing.com", 1])
end
it "backfills from the earliest event date on the first run when rollups are empty" do
@@ -183,26 +198,43 @@ RSpec.describe Jobs::MaintainBrowserPageviewRollups do
expect(event.normalized_referrer_version).to eq(BrowserPageviewReferrerInspector::VERSION)
end
- it "only backfills referrers from the active source" do
+ it "only backfills referrers from the source that applies on the event's date" do
+ freeze_time(Time.utc(2026, 6, 20, 12))
SiteSetting.dashboard_improvements = true
- piggyback_event =
+ UpcomingChangeEvent.create!(
+ upcoming_change_name: "dashboard_improvements",
+ event_type: :manual_opt_in,
+ created_at: Time.utc(2026, 6, 10, 9),
+ )
+
+ pre_cutover_piggyback =
Fabricate(
:browser_pageview_event_with_unnormalized_referrer,
referrer: "https://www.google.com/",
- source: BrowserPageviewEvent::SOURCE_PIGGYBACK,
+ source: :piggyback,
+ created_at: Time.utc(2026, 6, 9, 10),
+ )
+ post_cutover_piggyback =
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: "https://www.bing.com/",
+ source: :piggyback,
+ created_at: Time.utc(2026, 6, 15, 10),
)
- beacon_event =
+ post_cutover_beacon =
Fabricate(
:browser_pageview_event_with_unnormalized_referrer,
referrer: "https://www.reddit.com/",
- source: BrowserPageviewEvent::SOURCE_BEACON,
+ source: :beacon,
+ created_at: Time.utc(2026, 6, 15, 10),
)
job.execute({})
- expect(piggyback_event.reload.normalized_referrer_version).to be_nil
- expect(beacon_event.reload.normalized_referrer).to eq("reddit.com")
- expect(beacon_event.normalized_referrer_version).to eq(
+ expect(pre_cutover_piggyback.reload.normalized_referrer).to eq("google.com")
+ expect(post_cutover_piggyback.reload.normalized_referrer_version).to be_nil
+ expect(post_cutover_beacon.reload.normalized_referrer).to eq("reddit.com")
+ expect(post_cutover_beacon.normalized_referrer_version).to eq(
BrowserPageviewReferrerInspector::VERSION,
)
end
diff --git a/spec/models/browser_pageview_event_spec.rb b/spec/models/browser_pageview_event_spec.rb
index 01f9374cdf5..30fcb11d9ff 100644
--- a/spec/models/browser_pageview_event_spec.rb
+++ b/spec/models/browser_pageview_event_spec.rb
@@ -93,6 +93,46 @@ RSpec.describe BrowserPageviewEvent do
end
end
+ describe ".rollup_source_condition" do
+ it "matches only piggyback events when there is no cutover date" do
+ SiteSetting.dashboard_improvements = false
+ piggyback = Fabricate(:browser_pageview_event, source: :piggyback)
+ Fabricate(:browser_pageview_event, source: :beacon)
+
+ expect(described_class.where(described_class.rollup_source_condition)).to contain_exactly(
+ piggyback,
+ )
+ end
+
+ it "matches piggyback events before the cutover date and beacon events from it onwards" do
+ SiteSetting.dashboard_improvements = true
+ UpcomingChangeEvent.create!(
+ upcoming_change_name: "dashboard_improvements",
+ event_type: :manual_opt_in,
+ created_at: Time.utc(2026, 6, 10, 9),
+ )
+
+ pre_piggyback =
+ Fabricate(:browser_pageview_event, source: :piggyback, created_at: Time.utc(2026, 6, 9))
+ Fabricate(:browser_pageview_event, source: :beacon, created_at: Time.utc(2026, 6, 9))
+ Fabricate(:browser_pageview_event, source: :piggyback, created_at: Time.utc(2026, 6, 15))
+ post_beacon =
+ Fabricate(:browser_pageview_event, source: :beacon, created_at: Time.utc(2026, 6, 15))
+
+ expect(described_class.where(described_class.rollup_source_condition)).to contain_exactly(
+ pre_piggyback,
+ post_beacon,
+ )
+ end
+
+ it "qualifies columns with the given table alias" do
+ SiteSetting.dashboard_improvements = false
+ condition = described_class.rollup_source_condition(table: "e")
+
+ expect(condition).to eq("e.source = #{described_class::SOURCE_PIGGYBACK}")
+ end
+ end
+
it "truncates string fields before saving" do
event =
described_class.create!(
diff --git a/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb b/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb
index e497d09a1ad..9019e86adf6 100644
--- a/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb
+++ b/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb
@@ -157,6 +157,11 @@ RSpec.describe BrowserPageviewSessionEngagementDailyRollup do
it "counts only rollup-source pageviews, even within a single session" do
SiteSetting.dashboard_improvements = true
+ UpcomingChangeEvent.create!(
+ upcoming_change_name: "dashboard_improvements",
+ event_type: :manual_opt_in,
+ created_at: Time.utc(2026, 6, 1, 9),
+ )
beacon_event =
Fabricate(:browser_pageview_event, source: :beacon, created_at: Time.utc(2026, 6, 10, 9))
Fabricate(
diff --git a/spec/requests/admin/dashboard_controller_spec.rb b/spec/requests/admin/dashboard_controller_spec.rb
index f2008b4e58c..1ed4843d898 100644
--- a/spec/requests/admin/dashboard_controller_spec.rb
+++ b/spec/requests/admin/dashboard_controller_spec.rb
@@ -262,6 +262,11 @@ RSpec.describe Admin::DashboardController do
country_code = "US"
normalized_referrer = "sensitive-referrer.example"
event_date = Time.zone.local(2026, 5, 2, 12)
+ UpcomingChangeEvent.create!(
+ upcoming_change_name: "dashboard_improvements",
+ event_type: :manual_opt_in,
+ created_at: Time.zone.local(2026, 5, 1, 9),
+ )
2.times do
Fabricate(
diff --git a/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb b/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb
index 33c63c76054..0dac9b64b26 100644
--- a/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb
+++ b/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb
@@ -158,6 +158,11 @@ describe "Admin Dashboard Redesign | Site Traffic section" do
before do
SiteSetting.persist_browser_pageview_events = true
+ UpcomingChangeEvent.create!(
+ upcoming_change_name: "dashboard_improvements",
+ event_type: :manual_opt_in,
+ created_at: Time.zone.local(2026, 4, 30, 9),
+ )
Discourse.stubs(:current_hostname).returns("test.localhost")
Discourse.cache.clear
end