A platform for community discussion. Free, open, simple.
FIX: Split browser pageview rollup source at the cutover date (#42013)
Krzysztof Kotlarek820ebddd65ab
diff --git a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb b/app/jobs/scheduled/maintain_browser_pageview_rollups.rbindex 0147e824417..c7a4a68d67d 100644--- a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb+++ b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb@@ -50,7 +50,7 @@ module Jobsif BrowserPageviewCountryDailyRollup.none? && BrowserPageviewReferrerDailyRollup.none?earliest_event_date =BrowserPageviewEvent- .where(source: BrowserPageviewEvent.rollup_source)+ .where(BrowserPageviewEvent.rollup_source_condition).minimum(:created_at)&.to_date[earliest_event_date, end_date]@@ -71,11 +71,7 @@ module Jobsenddef next_batch- params = {- source: BrowserPageviewEvent.rollup_source,- version: BrowserPageviewReferrerInspector::VERSION,- limit: batch_size,- }+ params = { version: BrowserPageviewReferrerInspector::VERSION, limit: batch_size }retention_clause = ""if SiteSetting.clean_up_browser_pageview_events@@ -90,7 +86,7 @@ module JobsSELECT id, referrerFROM browser_pageview_eventsWHERE referrer IS NOT NULL- AND source = :source+ AND #{BrowserPageviewEvent.rollup_source_condition}AND (normalized_referrer_version IS NULLOR normalized_referrer_version < :version@@ -117,11 +113,7 @@ module Jobsenddef recomputable_dates(ids)- params = {- ids: ids,- source: BrowserPageviewEvent.rollup_source,- version: BrowserPageviewReferrerInspector::VERSION,- }+ params = { ids: ids, version: BrowserPageviewReferrerInspector::VERSION }retention_clause = ""if SiteSetting.clean_up_browser_pageview_events@@ -145,7 +137,7 @@ module JobsFROM browser_pageview_events eWHERE e.created_at >= touched_dates.dateAND e.created_at < touched_dates.date + 1- AND e.source = :source+ AND #{BrowserPageviewEvent.rollup_source_condition(table: "e")}AND e.referrer IS NOT NULLAND NOT EXISTS (SELECT 1diff --git a/app/models/browser_pageview_country_daily_rollup.rb b/app/models/browser_pageview_country_daily_rollup.rbindex eca37ce1289..d7f3e7a29cb 100644--- a/app/models/browser_pageview_country_daily_rollup.rb+++ b/app/models/browser_pageview_country_daily_rollup.rb@@ -1,11 +1,11 @@# frozen_string_literal: trueclass BrowserPageviewCountryDailyRollup < ActiveRecord::Base- def self.aggregate(start_date:, end_date:, source: BrowserPageviewEvent.rollup_source)+ def self.aggregate(start_date:, end_date:)start_date = start_date.to_dateend_date = end_date.to_date + 1- DB.exec(<<~SQL, start_date:, end_date:, source:)+ DB.exec(<<~SQL, start_date:, end_date:)INSERT INTO browser_pageview_country_daily_rollups (date, country_code, count, logged_in_count)SELECTcreated_at::date AS date,@@ -15,7 +15,7 @@ class BrowserPageviewCountryDailyRollup < ActiveRecord::BaseFROM browser_pageview_eventsWHERE created_at >= :start_dateAND created_at < :end_date- AND source = :source+ AND #{BrowserPageviewEvent.rollup_source_condition}GROUP BY date, country_codeON CONFLICT (date, country_code) DO UPDATESET count = EXCLUDED.count,diff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rbindex 089fac915f6..7b813bfaf28 100644--- a/app/models/browser_pageview_event.rb+++ b/app/models/browser_pageview_event.rb@@ -212,12 +212,19 @@ class BrowserPageviewEvent < ActiveRecord::BaseRETENTION_PERIOD.ago.beginning_of_dayend- def self.rollup_source- if UpcomingChanges.enabled?(:dashboard_improvements)- SOURCE_BEACON- else- SOURCE_PIGGYBACK- end+ def self.rollup_source_condition(table: nil)+ prefix = table ? "#{table}." : ""+ cutover_date = beacon_cutover_date+ return "#{prefix}source = #{SOURCE_PIGGYBACK}" if cutover_date.nil?++ sanitize_sql_array(+ [+ "(#{prefix}created_at < ? AND #{prefix}source = #{SOURCE_PIGGYBACK} " \+ "OR #{prefix}created_at >= ? AND #{prefix}source = #{SOURCE_BEACON})",+ cutover_date,+ cutover_date,+ ],+ )endbefore_save :truncate_fieldsdiff --git a/app/models/browser_pageview_referrer_daily_rollup.rb b/app/models/browser_pageview_referrer_daily_rollup.rbindex 1d24649510c..5c8d1e3c51f 100644--- a/app/models/browser_pageview_referrer_daily_rollup.rb+++ b/app/models/browser_pageview_referrer_daily_rollup.rb@@ -1,11 +1,11 @@# frozen_string_literal: trueclass BrowserPageviewReferrerDailyRollup < ActiveRecord::Base- def self.aggregate(start_date:, end_date:, source: BrowserPageviewEvent.rollup_source)+ def self.aggregate(start_date:, end_date:)start_date = start_date.to_dateend_date = end_date.to_date + 1- DB.exec(<<~SQL, start_date:, end_date:, source:)+ DB.exec(<<~SQL, start_date:, end_date:)INSERT INTO browser_pageview_referrer_daily_rollups (date, normalized_referrer, count, logged_in_count)SELECTcreated_at::date AS date,@@ -15,7 +15,7 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::BaseFROM browser_pageview_eventsWHERE created_at >= :start_dateAND created_at < :end_date- AND source = :source+ AND #{BrowserPageviewEvent.rollup_source_condition}GROUP BY date, normalized_referrerON CONFLICT (date, normalized_referrer) DO UPDATESET count = EXCLUDED.count,@@ -27,13 +27,11 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::Basedates = Array(dates).map(&:to_date).uniqreturn if dates.empty?- source = BrowserPageviewEvent.rollup_source-# The rollups are the permanent record, but their source events are pruned# after a retention period (CleanUpBrowserPageviewEvents). Only rebuild# dates that still have events so we never delete a rollup we can no longer# reconstruct from events.- dates = DB.query_single(<<~SQL, dates:, source:)+ dates = DB.query_single(<<~SQL, dates:)SELECT d.dateFROM unnest(ARRAY[:dates]::date[]) AS d(date)WHERE EXISTS (@@ -41,7 +39,7 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::BaseFROM browser_pageview_events eWHERE e.created_at >= d.dateAND e.created_at < d.date + 1- AND e.source = :source+ AND #{BrowserPageviewEvent.rollup_source_condition(table: "e")})SQLreturn if dates.empty?@@ -52,7 +50,7 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::BaseWHERE date IN (:dates)SQL- dates.each { |date| aggregate(start_date: date, end_date: date, source:) }+ dates.each { |date| aggregate(start_date: date, end_date: date) }endendenddiff --git a/app/models/browser_pageview_session_engagement_daily_rollup.rb b/app/models/browser_pageview_session_engagement_daily_rollup.rbindex 0890023cdd6..def526eb5eb 100644--- a/app/models/browser_pageview_session_engagement_daily_rollup.rb+++ b/app/models/browser_pageview_session_engagement_daily_rollup.rb@@ -5,12 +5,13 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::BaseMIN_SESSION_AGE = 10.minutesprivate_constant :BOUNCE_ENGAGED_SECONDS_THRESHOLD, :MIN_SESSION_AGE- def self.aggregate(start_date:, end_date:, source: BrowserPageviewEvent.rollup_source)+ def self.aggregate(start_date:, end_date:)start_date = start_date.to_dateend_date = end_date.to_date + 1+ source_condition = BrowserPageviewEvent.rollup_source_conditiontransaction do- DB.exec(<<~SQL, start_date:, end_date:, source:)+ DB.exec(<<~SQL, start_date:, end_date:)DELETE FROM browser_pageview_session_engagement_daily_rollups rollupWHERE rollup.date >= :start_dateAND rollup.date < :end_date@@ -19,7 +20,7 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::BaseFROM browser_pageview_eventsWHERE created_at >= rollup.dateAND created_at < rollup.date + 1- AND source = :source+ AND #{source_condition})SQL@@ -30,7 +31,7 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::BaseFROM browser_pageview_eventsWHERE created_at >= :start_dateAND created_at < LEAST(:end_date::timestamp, :session_started_before::timestamp)- AND source = :source+ AND #{source_condition}),session_pageviews AS (SELECT@@ -40,7 +41,7 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Basebool_or(bpe.user_id IS NOT NULL) AS logged_inFROM browser_pageview_events bpeJOIN active_sessions ON active_sessions.session_id = bpe.session_id- WHERE bpe.source = :source+ WHERE #{BrowserPageviewEvent.rollup_source_condition(table: "bpe")}GROUP BY bpe.session_idHAVING MIN(bpe.created_at) >= :start_date)@@ -64,7 +65,6 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Baseend_date:,session_started_before: MIN_SESSION_AGE.ago,bounce_threshold: BOUNCE_ENGAGED_SECONDS_THRESHOLD,- source:,)endenddiff --git a/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb b/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rbindex 5ef71dffbe1..0426a0f6241 100644--- a/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb+++ b/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb@@ -463,7 +463,7 @@ module DiscourseAiAND e.created_at < (:end_date::date + 1)AND e.topic_id IS NOT NULLAND e.normalized_referrer IS NOT NULL- AND e.source = :source+ AND #{BrowserPageviewEvent.rollup_source_condition(table: "e")}AND #{topic_conditions}GROUP BY e.topic_id, t.titleORDER BY visits DESC@@ -471,7 +471,6 @@ module DiscourseAiSQLstart_date: @start_date,end_date: @end_date,- source: BrowserPageviewEvent.rollup_source,category_ids: scoped_category_ids,).firstreturn if row.nil? || row.visits.to_i < 50diff --git a/spec/jobs/maintain_browser_pageview_rollups_spec.rb b/spec/jobs/maintain_browser_pageview_rollups_spec.rbindex 4a6e332c912..b62f225cdaa 100644--- a/spec/jobs/maintain_browser_pageview_rollups_spec.rb+++ b/spec/jobs/maintain_browser_pageview_rollups_spec.rb@@ -34,38 +34,53 @@ RSpec.describe Jobs::MaintainBrowserPageviewRollups do).to eq(1)end- it "uses beacon source for rollups when dashboard_improvements is enabled" do- SiteSetting.dashboard_improvements = false+ it "rolls up piggyback events before the beacon cutover date and beacon events after it" do+ freeze_time(Time.utc(2026, 6, 20, 12))+ SiteSetting.dashboard_improvements = true+ UpcomingChangeEvent.create!(+ upcoming_change_name: "dashboard_improvements",+ event_type: :manual_opt_in,+ created_at: Time.utc(2026, 6, 10, 9),+ )+Fabricate(:browser_pageview_event,country_code: "US",normalized_referrer: "google.com",- source: BrowserPageviewEvent::SOURCE_PIGGYBACK,+ source: :piggyback,+ created_at: Time.utc(2026, 6, 9, 10),)Fabricate(:browser_pageview_event,country_code: "GB",normalized_referrer: "reddit.com",- source: BrowserPageviewEvent::SOURCE_BEACON,+ source: :beacon,+ created_at: Time.utc(2026, 6, 9, 10),)-- job.execute({})-- expect(BrowserPageviewCountryDailyRollup.pluck(:country_code, :count)).to eq([["US", 1]])- expect(BrowserPageviewReferrerDailyRollup.pluck(:normalized_referrer, :count)).to eq(- [["google.com", 1]],+ Fabricate(+ :browser_pageview_event,+ country_code: "FR",+ normalized_referrer: "bing.com",+ source: :beacon,+ created_at: Time.utc(2026, 6, 15, 10),+ )+ Fabricate(+ :browser_pageview_event,+ country_code: "DE",+ normalized_referrer: "duckduckgo.com",+ source: :piggyback,+ created_at: Time.utc(2026, 6, 15, 10),)- SiteSetting.dashboard_improvements = truejob.execute({})expect(BrowserPageviewCountryDailyRollup.pluck(:country_code, :count)).to contain_exactly(["US", 1],- ["GB", 1],+ ["FR", 1],)expect(BrowserPageviewReferrerDailyRollup.pluck(:normalized_referrer, :count),- ).to contain_exactly(["google.com", 1], ["reddit.com", 1])+ ).to contain_exactly(["google.com", 1], ["bing.com", 1])endit "backfills from the earliest event date on the first run when rollups are empty" do@@ -183,26 +198,43 @@ RSpec.describe Jobs::MaintainBrowserPageviewRollups doexpect(event.normalized_referrer_version).to eq(BrowserPageviewReferrerInspector::VERSION)end- it "only backfills referrers from the active source" do+ it "only backfills referrers from the source that applies on the event's date" do+ freeze_time(Time.utc(2026, 6, 20, 12))SiteSetting.dashboard_improvements = true- piggyback_event =+ UpcomingChangeEvent.create!(+ upcoming_change_name: "dashboard_improvements",+ event_type: :manual_opt_in,+ created_at: Time.utc(2026, 6, 10, 9),+ )++ pre_cutover_piggyback =Fabricate(:browser_pageview_event_with_unnormalized_referrer,referrer: "https://www.google.com/",- source: BrowserPageviewEvent::SOURCE_PIGGYBACK,+ source: :piggyback,+ created_at: Time.utc(2026, 6, 9, 10),+ )+ post_cutover_piggyback =+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: "https://www.bing.com/",+ source: :piggyback,+ created_at: Time.utc(2026, 6, 15, 10),)- beacon_event =+ post_cutover_beacon =Fabricate(:browser_pageview_event_with_unnormalized_referrer,referrer: "https://www.reddit.com/",- source: BrowserPageviewEvent::SOURCE_BEACON,+ source: :beacon,+ created_at: Time.utc(2026, 6, 15, 10),)job.execute({})- expect(piggyback_event.reload.normalized_referrer_version).to be_nil- expect(beacon_event.reload.normalized_referrer).to eq("reddit.com")- expect(beacon_event.normalized_referrer_version).to eq(+ expect(pre_cutover_piggyback.reload.normalized_referrer).to eq("google.com")+ expect(post_cutover_piggyback.reload.normalized_referrer_version).to be_nil+ expect(post_cutover_beacon.reload.normalized_referrer).to eq("reddit.com")+ expect(post_cutover_beacon.normalized_referrer_version).to eq(BrowserPageviewReferrerInspector::VERSION,)enddiff --git a/spec/models/browser_pageview_event_spec.rb b/spec/models/browser_pageview_event_spec.rbindex 01f9374cdf5..30fcb11d9ff 100644--- a/spec/models/browser_pageview_event_spec.rb+++ b/spec/models/browser_pageview_event_spec.rb@@ -93,6 +93,46 @@ RSpec.describe BrowserPageviewEvent doendend+ describe ".rollup_source_condition" do+ it "matches only piggyback events when there is no cutover date" do+ SiteSetting.dashboard_improvements = false+ piggyback = Fabricate(:browser_pageview_event, source: :piggyback)+ Fabricate(:browser_pageview_event, source: :beacon)++ expect(described_class.where(described_class.rollup_source_condition)).to contain_exactly(+ piggyback,+ )+ end++ it "matches piggyback events before the cutover date and beacon events from it onwards" do+ SiteSetting.dashboard_improvements = true+ UpcomingChangeEvent.create!(+ upcoming_change_name: "dashboard_improvements",+ event_type: :manual_opt_in,+ created_at: Time.utc(2026, 6, 10, 9),+ )++ pre_piggyback =+ Fabricate(:browser_pageview_event, source: :piggyback, created_at: Time.utc(2026, 6, 9))+ Fabricate(:browser_pageview_event, source: :beacon, created_at: Time.utc(2026, 6, 9))+ Fabricate(:browser_pageview_event, source: :piggyback, created_at: Time.utc(2026, 6, 15))+ post_beacon =+ Fabricate(:browser_pageview_event, source: :beacon, created_at: Time.utc(2026, 6, 15))++ expect(described_class.where(described_class.rollup_source_condition)).to contain_exactly(+ pre_piggyback,+ post_beacon,+ )+ end++ it "qualifies columns with the given table alias" do+ SiteSetting.dashboard_improvements = false+ condition = described_class.rollup_source_condition(table: "e")++ expect(condition).to eq("e.source = #{described_class::SOURCE_PIGGYBACK}")+ end+ end+it "truncates string fields before saving" doevent =described_class.create!(diff --git a/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb b/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rbindex e497d09a1ad..9019e86adf6 100644--- a/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb+++ b/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb@@ -157,6 +157,11 @@ RSpec.describe BrowserPageviewSessionEngagementDailyRollup doit "counts only rollup-source pageviews, even within a single session" doSiteSetting.dashboard_improvements = true+ UpcomingChangeEvent.create!(+ upcoming_change_name: "dashboard_improvements",+ event_type: :manual_opt_in,+ created_at: Time.utc(2026, 6, 1, 9),+ )beacon_event =Fabricate(:browser_pageview_event, source: :beacon, created_at: Time.utc(2026, 6, 10, 9))Fabricate(diff --git a/spec/requests/admin/dashboard_controller_spec.rb b/spec/requests/admin/dashboard_controller_spec.rbindex f2008b4e58c..1ed4843d898 100644--- a/spec/requests/admin/dashboard_controller_spec.rb+++ b/spec/requests/admin/dashboard_controller_spec.rb@@ -262,6 +262,11 @@ RSpec.describe Admin::DashboardController docountry_code = "US"normalized_referrer = "sensitive-referrer.example"event_date = Time.zone.local(2026, 5, 2, 12)+ UpcomingChangeEvent.create!(+ upcoming_change_name: "dashboard_improvements",+ event_type: :manual_opt_in,+ created_at: Time.zone.local(2026, 5, 1, 9),+ )2.times doFabricate(diff --git a/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb b/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rbindex 33c63c76054..0dac9b64b26 100644--- a/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb+++ b/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb@@ -158,6 +158,11 @@ describe "Admin Dashboard Redesign | Site Traffic section" dobefore doSiteSetting.persist_browser_pageview_events = true+ UpcomingChangeEvent.create!(+ upcoming_change_name: "dashboard_improvements",+ event_type: :manual_opt_in,+ created_at: Time.zone.local(2026, 4, 30, 9),+ )Discourse.stubs(:current_hostname).returns("test.localhost")Discourse.cache.clearend