A platform for community discussion. Free, open, simple.
PERF: Daily rollups for top countries and top referrers reports (#40285)
Alan Guo Xiang Tan2cc3ac34e73b
diff --git a/app/jobs/scheduled/aggregate_browser_pageview_daily_rollups.rb b/app/jobs/scheduled/aggregate_browser_pageview_daily_rollups.rbnew file mode 100644index 00000000000..cc34d0bd759--- /dev/null+++ b/app/jobs/scheduled/aggregate_browser_pageview_daily_rollups.rb@@ -0,0 +1,37 @@+# frozen_string_literal: true++module Jobs+ class AggregateBrowserPageviewDailyRollups < ::Jobs::Scheduled+ every 30.minutes++ LOCK_KEY = "aggregate_browser_pageview_daily_rollups"++ def execute(args)+ return if !SiteSetting.persist_browser_pageview_events++ start_date, end_date = aggregation_window+ return if start_date.nil?++ DistributedMutex.synchronize(LOCK_KEY, validity: 10.minutes) do+ BrowserPageviewCountryDailyRollup.aggregate(start_date: start_date, end_date: end_date)+ BrowserPageviewReferrerDailyRollup.aggregate(start_date: start_date, end_date: end_date)+ end++ Report.clear_cache("top_countries_by_browser_pageviews")+ Report.clear_cache("top_referrers_by_browser_pageviews")+ end++ private++ def aggregation_window+ end_date = Time.zone.today++ if BrowserPageviewCountryDailyRollup.none? && BrowserPageviewReferrerDailyRollup.none?+ earliest_event_date = BrowserPageviewEvent.minimum(:created_at)&.to_date+ [earliest_event_date, end_date]+ else+ [1.day.ago.to_date, end_date]+ end+ end+ end+enddiff --git a/app/models/browser_pageview_country_daily_rollup.rb b/app/models/browser_pageview_country_daily_rollup.rbnew file mode 100644index 00000000000..60cd54a4e34--- /dev/null+++ b/app/models/browser_pageview_country_daily_rollup.rb@@ -0,0 +1,36 @@+# frozen_string_literal: true++class BrowserPageviewCountryDailyRollup < ActiveRecord::Base+ def self.aggregate(start_date:, end_date:)+ DB.exec(<<~SQL, start_date: start_date.to_date, end_date: end_date.to_date + 1)+ INSERT INTO browser_pageview_country_daily_rollups (date, country_code, count, logged_in_count)+ SELECT+ created_at::date AS date,+ country_code,+ COUNT(*) AS count,+ COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS logged_in_count+ FROM browser_pageview_events+ WHERE created_at >= :start_date+ AND created_at < :end_date+ GROUP BY date, country_code+ ON CONFLICT (date, country_code) DO UPDATE+ SET count = EXCLUDED.count,+ logged_in_count = EXCLUDED.logged_in_count+ SQL+ end+end++# == Schema Information+#+# Table name: browser_pageview_country_daily_rollups+#+# id :bigint not null, primary key+# count :bigint not null+# country_code :string(2)+# date :date not null+# logged_in_count :bigint not null+#+# Indexes+#+# idx_bpcd_rollups_date_country_unique (date,country_code) UNIQUE NULLS NOT DISTINCT+#diff --git a/app/models/browser_pageview_referrer_daily_rollup.rb b/app/models/browser_pageview_referrer_daily_rollup.rbnew file mode 100644index 00000000000..e210d23172d--- /dev/null+++ b/app/models/browser_pageview_referrer_daily_rollup.rb@@ -0,0 +1,36 @@+# frozen_string_literal: true++class BrowserPageviewReferrerDailyRollup < ActiveRecord::Base+ def self.aggregate(start_date:, end_date:)+ DB.exec(<<~SQL, start_date: start_date.to_date, end_date: end_date.to_date + 1)+ INSERT INTO browser_pageview_referrer_daily_rollups (date, normalized_referrer, count, logged_in_count)+ SELECT+ created_at::date AS date,+ normalized_referrer,+ COUNT(*) AS count,+ COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS logged_in_count+ FROM browser_pageview_events+ WHERE created_at >= :start_date+ AND created_at < :end_date+ GROUP BY date, normalized_referrer+ ON CONFLICT (date, normalized_referrer) DO UPDATE+ SET count = EXCLUDED.count,+ logged_in_count = EXCLUDED.logged_in_count+ SQL+ end+end++# == Schema Information+#+# Table name: browser_pageview_referrer_daily_rollups+#+# id :bigint not null, primary key+# count :bigint not null+# date :date not null+# logged_in_count :bigint not null+# normalized_referrer :string(2000)+#+# Indexes+#+# idx_bprd_rollups_date_referrer_unique (date,normalized_referrer) UNIQUE NULLS NOT DISTINCT+#diff --git a/app/models/concerns/reports/top_countries_by_browser_pageviews.rb b/app/models/concerns/reports/top_countries_by_browser_pageviews.rbindex 0d7cfb6c1f4..75dfc88892e 100644--- a/app/models/concerns/reports/top_countries_by_browser_pageviews.rb+++ b/app/models/concerns/reports/top_countries_by_browser_pageviews.rb@@ -22,20 +22,20 @@ module Reports::TopCountriesByBrowserPageviews},]- user_filter_sql = SiteSetting.login_required ? "AND user_id IS NOT NULL" : ""+ count_expr = SiteSetting.login_required ? "logged_in_count" : "count"end_date_exclusive = report.end_date.to_date + 1sql = <<~SQLWITH ranked AS (SELECTcountry_code,- COUNT(*) AS count,- SUM(COUNT(*)) OVER () AS total- FROM browser_pageview_events- WHERE created_at >= :start_date- AND created_at < :end_date_exclusive- #{user_filter_sql}+ SUM(#{count_expr}) AS count,+ SUM(SUM(#{count_expr})) OVER () AS total+ FROM browser_pageview_country_daily_rollups+ WHERE date >= :start_date+ AND date < :end_date_exclusiveGROUP BY country_code+ HAVING SUM(#{count_expr}) > 0)SELECT country_code, count,CASE WHEN total = 0 THEN 0diff --git a/app/models/concerns/reports/top_referrers_by_browser_pageviews.rb b/app/models/concerns/reports/top_referrers_by_browser_pageviews.rbindex b1858fe4105..6eec87d775e 100644--- a/app/models/concerns/reports/top_referrers_by_browser_pageviews.rb+++ b/app/models/concerns/reports/top_referrers_by_browser_pageviews.rb@@ -20,7 +20,7 @@ module Reports::TopReferrersByBrowserPageviews},]- user_filter_sql = SiteSetting.login_required ? "AND user_id IS NOT NULL" : ""+ count_expr = SiteSetting.login_required ? "logged_in_count" : "count"end_date_exclusive = report.end_date.to_date + 1host = BrowserPageviewReferrerInspector.normalize_host(Discourse.current_hostname)@@ -30,12 +30,11 @@ module Reports::TopReferrersByBrowserPageviewsWITH ranked AS (SELECTnormalized_referrer,- COUNT(*) AS count,- SUM(COUNT(*)) OVER () AS total- FROM browser_pageview_events- WHERE created_at >= :start_date- AND created_at < :end_date_exclusive- #{user_filter_sql}+ SUM(#{count_expr}) AS count,+ SUM(SUM(#{count_expr})) OVER () AS total+ FROM browser_pageview_referrer_daily_rollups+ WHERE date >= :start_date+ AND date < :end_date_exclusiveAND (normalized_referrer IS NULLOR (@@ -45,6 +44,7 @@ module Reports::TopReferrersByBrowserPageviews))GROUP BY normalized_referrer+ HAVING SUM(#{count_expr}) > 0)SELECT normalized_referrer, count,CASE WHEN total = 0 THEN 0diff --git a/db/migrate/20260525105006_create_browser_pageview_country_daily_rollups.rb b/db/migrate/20260525105006_create_browser_pageview_country_daily_rollups.rbnew file mode 100644index 00000000000..f7c524d6aeb--- /dev/null+++ b/db/migrate/20260525105006_create_browser_pageview_country_daily_rollups.rb@@ -0,0 +1,17 @@+# frozen_string_literal: true+class CreateBrowserPageviewCountryDailyRollups < ActiveRecord::Migration[8.0]+ def change+ create_table :browser_pageview_country_daily_rollups do |t|+ t.date :date, null: false+ t.string :country_code, limit: 2+ t.bigint :count, null: false+ t.bigint :logged_in_count, null: false+ end++ add_index :browser_pageview_country_daily_rollups,+ %i[date country_code],+ unique: true,+ nulls_not_distinct: true,+ name: "idx_bpcd_rollups_date_country_unique"+ end+enddiff --git a/db/migrate/20260525105009_create_browser_pageview_referrer_daily_rollups.rb b/db/migrate/20260525105009_create_browser_pageview_referrer_daily_rollups.rbnew file mode 100644index 00000000000..053f11abca7--- /dev/null+++ b/db/migrate/20260525105009_create_browser_pageview_referrer_daily_rollups.rb@@ -0,0 +1,17 @@+# frozen_string_literal: true+class CreateBrowserPageviewReferrerDailyRollups < ActiveRecord::Migration[8.0]+ def change+ create_table :browser_pageview_referrer_daily_rollups do |t|+ t.date :date, null: false+ t.string :normalized_referrer, limit: 2000+ t.bigint :count, null: false+ t.bigint :logged_in_count, null: false+ end++ add_index :browser_pageview_referrer_daily_rollups,+ %i[date normalized_referrer],+ unique: true,+ nulls_not_distinct: true,+ name: "idx_bprd_rollups_date_referrer_unique"+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex 88a26a63a9f..0f6d2118587 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -1833,6 +1833,38 @@ CREATE SEQUENCE public.bookmarks_id_seqALTER SEQUENCE public.bookmarks_id_seq OWNED BY public.bookmarks.id;+--+-- Name: browser_pageview_country_daily_rollups; Type: TABLE; Schema: public; Owner: -+--++CREATE TABLE public.browser_pageview_country_daily_rollups (+ id bigint NOT NULL,+ date date NOT NULL,+ country_code character varying(2),+ count bigint NOT NULL,+ logged_in_count bigint NOT NULL+);+++--+-- Name: browser_pageview_country_daily_rollups_id_seq; Type: SEQUENCE; Schema: public; Owner: -+--++CREATE SEQUENCE public.browser_pageview_country_daily_rollups_id_seq+ START WITH 1+ INCREMENT BY 1+ NO MINVALUE+ NO MAXVALUE+ CACHE 1;+++--+-- Name: browser_pageview_country_daily_rollups_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -+--++ALTER SEQUENCE public.browser_pageview_country_daily_rollups_id_seq OWNED BY public.browser_pageview_country_daily_rollups.id;++---- Name: browser_pageview_event_scores; Type: TABLE; Schema: public; Owner: ---@@ -1908,6 +1940,38 @@ CREATE SEQUENCE public.browser_pageview_events_id_seqALTER SEQUENCE public.browser_pageview_events_id_seq OWNED BY public.browser_pageview_events.id;+--+-- Name: browser_pageview_referrer_daily_rollups; Type: TABLE; Schema: public; Owner: -+--++CREATE TABLE public.browser_pageview_referrer_daily_rollups (+ id bigint NOT NULL,+ date date NOT NULL,+ normalized_referrer character varying(2000),+ count bigint NOT NULL,+ logged_in_count bigint NOT NULL+);+++--+-- Name: browser_pageview_referrer_daily_rollups_id_seq; Type: SEQUENCE; Schema: public; Owner: -+--++CREATE SEQUENCE public.browser_pageview_referrer_daily_rollups_id_seq+ START WITH 1+ INCREMENT BY 1+ NO MINVALUE+ NO MAXVALUE+ CACHE 1;+++--+-- Name: browser_pageview_referrer_daily_rollups_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -+--++ALTER SEQUENCE public.browser_pageview_referrer_daily_rollups_id_seq OWNED BY public.browser_pageview_referrer_daily_rollups.id;++---- Name: calendar_events; Type: TABLE; Schema: public; Owner: ---@@ -11592,6 +11656,13 @@ ALTER TABLE ONLY public.badges ALTER COLUMN id SET DEFAULT nextval('public.badgeALTER TABLE ONLY public.bookmarks ALTER COLUMN id SET DEFAULT nextval('public.bookmarks_id_seq'::regclass);+--+-- Name: browser_pageview_country_daily_rollups id; Type: DEFAULT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.browser_pageview_country_daily_rollups ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_country_daily_rollups_id_seq'::regclass);++---- Name: browser_pageview_event_scores id; Type: DEFAULT; Schema: public; Owner: ---@@ -11606,6 +11677,13 @@ ALTER TABLE ONLY public.browser_pageview_event_scores ALTER COLUMN id SET DEFAULALTER TABLE ONLY public.browser_pageview_events ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_events_id_seq'::regclass);+--+-- Name: browser_pageview_referrer_daily_rollups id; Type: DEFAULT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.browser_pageview_referrer_daily_rollups ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_referrer_daily_rollups_id_seq'::regclass);++---- Name: calendar_events id; Type: DEFAULT; Schema: public; Owner: ---@@ -13671,6 +13749,14 @@ ALTER TABLE ONLY public.bookmarksADD CONSTRAINT bookmarks_pkey PRIMARY KEY (id);+--+-- Name: browser_pageview_country_daily_rollups browser_pageview_country_daily_rollups_pkey; Type: CONSTRAINT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.browser_pageview_country_daily_rollups+ ADD CONSTRAINT browser_pageview_country_daily_rollups_pkey PRIMARY KEY (id);++---- Name: browser_pageview_event_scores browser_pageview_event_scores_pkey; Type: CONSTRAINT; Schema: public; Owner: ---@@ -13687,6 +13773,14 @@ ALTER TABLE ONLY public.browser_pageview_eventsADD CONSTRAINT browser_pageview_events_pkey PRIMARY KEY (id);+--+-- Name: browser_pageview_referrer_daily_rollups browser_pageview_referrer_daily_rollups_pkey; Type: CONSTRAINT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.browser_pageview_referrer_daily_rollups+ ADD CONSTRAINT browser_pageview_referrer_daily_rollups_pkey PRIMARY KEY (id);++---- Name: calendar_events calendar_events_pkey; Type: CONSTRAINT; Schema: public; Owner: ---@@ -15889,6 +15983,13 @@ CREATE UNIQUE INDEX idx_ai_bot_conversation_stars_user_topic ON public.discourseCREATE UNIQUE INDEX idx_bookmarks_user_polymorphic_unique ON public.bookmarks USING btree (user_id, bookmarkable_type, bookmarkable_id);+--+-- Name: idx_bpcd_rollups_date_country_unique; Type: INDEX; Schema: public; Owner: -+--++CREATE UNIQUE INDEX idx_bpcd_rollups_date_country_unique ON public.browser_pageview_country_daily_rollups USING btree (date, country_code) NULLS NOT DISTINCT;++---- Name: idx_bpe_created_at_country_code; Type: INDEX; Schema: public; Owner: ---@@ -15917,6 +16018,13 @@ CREATE INDEX idx_bpe_ip_ua_created_at ON public.browser_pageview_events USING btCREATE INDEX idx_bpe_session_created_at ON public.browser_pageview_events USING btree (session_id, created_at);+--+-- Name: idx_bprd_rollups_date_referrer_unique; Type: INDEX; Schema: public; Owner: -+--++CREATE UNIQUE INDEX idx_bprd_rollups_date_referrer_unique ON public.browser_pageview_referrer_daily_rollups USING btree (date, normalized_referrer) NULLS NOT DISTINCT;++---- Name: idx_category_posting_review_groups_unique; Type: INDEX; Schema: public; Owner: ---@@ -20938,6 +21046,8 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groupsSET search_path TO "$user", public;INSERT INTO "schema_migrations" (version) VALUES+('20260525105009'),+('20260525105006'),('20260522043337'),('20260520090937'),('20260518104900'),diff --git a/spec/jobs/aggregate_browser_pageview_daily_rollups_spec.rb b/spec/jobs/aggregate_browser_pageview_daily_rollups_spec.rbnew file mode 100644index 00000000000..edbc9cbd967--- /dev/null+++ b/spec/jobs/aggregate_browser_pageview_daily_rollups_spec.rb@@ -0,0 +1,57 @@+# frozen_string_literal: true++RSpec.describe Jobs::AggregateBrowserPageviewDailyRollups do+ before { Discourse.stubs(:current_hostname).returns("forum.example.com") }++ it "does nothing when persist_browser_pageview_events is disabled" do+ SiteSetting.persist_browser_pageview_events = false+ Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")++ expect { described_class.new.execute({}) }.not_to change {+ BrowserPageviewCountryDailyRollup.count + BrowserPageviewReferrerDailyRollup.count+ }+ end++ it "aggregates both country and referrer rollups from recent pageview events" do+ SiteSetting.persist_browser_pageview_events = true+ Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")++ described_class.new.execute({})++ expect(BrowserPageviewCountryDailyRollup.where(country_code: "US").sum(:count)).to eq(1)+ expect(+ BrowserPageviewReferrerDailyRollup.where(normalized_referrer: "google.com").sum(:count),+ ).to eq(1)+ end++ it "backfills from the earliest event date on the first run when rollups are empty" do+ SiteSetting.persist_browser_pageview_events = true+ Fabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)+ Fabricate(:browser_pageview_event, country_code: "GB", created_at: 5.days.ago)++ described_class.new.execute({})++ expect(BrowserPageviewCountryDailyRollup.pluck(:country_code)).to contain_exactly("US", "GB")+ end++ it "clears the report cache after running so stale empty results do not linger" do+ SiteSetting.persist_browser_pageview_events = true+ Fabricate(:browser_pageview_event, country_code: "US")+ Report.expects(:clear_cache).with("top_countries_by_browser_pageviews").once+ Report.expects(:clear_cache).with("top_referrers_by_browser_pageviews").once++ described_class.new.execute({})+ end++ it "only aggregates yesterday and today once historical rollups are populated" do+ SiteSetting.persist_browser_pageview_events = true+ Fabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)+ described_class.new.execute({}) # first run backfills everything++ Fabricate(:browser_pageview_event, country_code: "GB", created_at: 60.days.ago) # late old event+ Fabricate(:browser_pageview_event, country_code: "FR") # today+ described_class.new.execute({}) # second run only does yesterday + today++ expect(BrowserPageviewCountryDailyRollup.pluck(:country_code)).to contain_exactly("US", "FR")+ end+enddiff --git a/spec/models/browser_pageview_country_daily_rollup_spec.rb b/spec/models/browser_pageview_country_daily_rollup_spec.rbnew file mode 100644index 00000000000..0564a6dde1c--- /dev/null+++ b/spec/models/browser_pageview_country_daily_rollup_spec.rb@@ -0,0 +1,85 @@+# frozen_string_literal: true++RSpec.describe BrowserPageviewCountryDailyRollup do+ describe ".aggregate" do+ let(:start_date) { 3.days.ago.to_date }+ let(:end_date) { Date.current }++ it "groups events by date and country_code into one row per combination" do+ yesterday = 1.day.ago+ today = Time.current+ Fabricate(:browser_pageview_event, country_code: "US", created_at: yesterday)+ Fabricate(:browser_pageview_event, country_code: "US", created_at: today)+ Fabricate(:browser_pageview_event, country_code: "GB", created_at: today)++ described_class.aggregate(start_date: start_date, end_date: end_date)++ rollups = described_class.order(:date, :country_code).pluck(:date, :country_code, :count)+ expect(rollups).to contain_exactly(+ [yesterday.to_date, "US", 1],+ [today.to_date, "US", 1],+ [today.to_date, "GB", 1],+ )+ end++ it "splits total count from logged-in count" do+ user = Fabricate(:user)+ Fabricate(:browser_pageview_event, country_code: "US", user_id: user.id)+ Fabricate(:browser_pageview_event, country_code: "US")+ Fabricate(:browser_pageview_event, country_code: "US")++ described_class.aggregate(start_date: start_date, end_date: end_date)++ row = described_class.first+ expect(row.count).to eq(3)+ expect(row.logged_in_count).to eq(1)+ end++ it "stores rows for NULL country_code (no GeoIP match) without duplication on re-run" do+ Fabricate(:browser_pageview_event, country_code: nil)+ Fabricate(:browser_pageview_event, country_code: nil)++ described_class.aggregate(start_date: start_date, end_date: end_date)+ described_class.aggregate(start_date: start_date, end_date: end_date)++ null_rows = described_class.where(country_code: nil)+ expect(null_rows.count).to eq(1)+ expect(null_rows.first.count).to eq(2)+ end++ it "is idempotent — running twice produces the same totals" do+ Fabricate(:browser_pageview_event, country_code: "US")+ Fabricate(:browser_pageview_event, country_code: "US")++ described_class.aggregate(start_date: start_date, end_date: end_date)+ described_class.aggregate(start_date: start_date, end_date: end_date)++ expect(described_class.where(country_code: "US").sum(:count)).to eq(2)+ end++ it "updates existing rollup rows when re-aggregating with new events" do+ Fabricate(:browser_pageview_event, country_code: "US")+ described_class.aggregate(start_date: start_date, end_date: end_date)++ Fabricate(:browser_pageview_event, country_code: "US")+ described_class.aggregate(start_date: start_date, end_date: end_date)++ expect(described_class.where(country_code: "US").sum(:count)).to eq(2)+ end++ it "only aggregates events within the requested date range" do+ Fabricate(:browser_pageview_event, country_code: "US", created_at: 10.days.ago)+ Fabricate(:browser_pageview_event, country_code: "US", created_at: 1.day.ago)++ described_class.aggregate(start_date: start_date, end_date: end_date)++ expect(described_class.sum(:count)).to eq(1)+ end++ it "is a no-op when no events exist in the range" do+ expect {+ described_class.aggregate(start_date: start_date, end_date: end_date)+ }.not_to change { described_class.count }+ end+ end+enddiff --git a/spec/models/browser_pageview_referrer_daily_rollup_spec.rb b/spec/models/browser_pageview_referrer_daily_rollup_spec.rbnew file mode 100644index 00000000000..c1d6ae6aee1--- /dev/null+++ b/spec/models/browser_pageview_referrer_daily_rollup_spec.rb@@ -0,0 +1,85 @@+# frozen_string_literal: true++RSpec.describe BrowserPageviewReferrerDailyRollup do+ describe ".aggregate" do+ let(:start_date) { 3.days.ago.to_date }+ let(:end_date) { Date.current }++ it "groups events by date and normalized_referrer into one row per combination" do+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com")+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com")+ Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com")++ described_class.aggregate(start_date: start_date, end_date: end_date)++ rollups = described_class.order(:normalized_referrer).pluck(:normalized_referrer, :count)+ expect(rollups).to contain_exactly(["google.com", 2], ["reddit.com", 1])+ end++ it "splits total count from logged-in count" do+ user = Fabricate(:user)+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com", user_id: user.id)+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com")++ described_class.aggregate(start_date: start_date, end_date: end_date)++ row = described_class.find_by(normalized_referrer: "google.com")+ expect(row.count).to eq(2)+ expect(row.logged_in_count).to eq(1)+ end++ it "stores NULL-referrer rows (direct visits)" do+ Fabricate(:browser_pageview_event, normalized_referrer: nil)+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com")++ described_class.aggregate(start_date: start_date, end_date: end_date)++ expect(described_class.where(normalized_referrer: nil).first.count).to eq(1)+ end++ it "stores internal-referrer rows so the data remains recoverable on hostname or rule changes" do+ Fabricate(:browser_pageview_event, normalized_referrer: "forum.example.com")+ Fabricate(:browser_pageview_event, normalized_referrer: "forum.example.com/t/topic/1")+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com")++ described_class.aggregate(start_date: start_date, end_date: end_date)++ expect(described_class.pluck(:normalized_referrer)).to contain_exactly(+ "forum.example.com",+ "forum.example.com/t/topic/1",+ "google.com",+ )+ end++ it "is idempotent — running twice produces the same totals" do+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com")+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com")++ described_class.aggregate(start_date: start_date, end_date: end_date)+ described_class.aggregate(start_date: start_date, end_date: end_date)++ expect(described_class.where(normalized_referrer: "google.com").sum(:count)).to eq(2)+ end++ it "is idempotent for NULL referrer rows" do+ Fabricate(:browser_pageview_event, normalized_referrer: nil)+ Fabricate(:browser_pageview_event, normalized_referrer: nil)++ described_class.aggregate(start_date: start_date, end_date: end_date)+ described_class.aggregate(start_date: start_date, end_date: end_date)++ null_rows = described_class.where(normalized_referrer: nil)+ expect(null_rows.count).to eq(1)+ expect(null_rows.first.count).to eq(2)+ end++ it "only aggregates events within the requested date range" do+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com", created_at: 10.days.ago)+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com", created_at: 1.day.ago)++ described_class.aggregate(start_date: start_date, end_date: end_date)++ expect(described_class.sum(:count)).to eq(1)+ end+ end+enddiff --git a/spec/models/reports/top_countries_by_browser_pageviews_spec.rb b/spec/models/reports/top_countries_by_browser_pageviews_spec.rbindex 13b7b967842..c13daceb7cd 100644--- a/spec/models/reports/top_countries_by_browser_pageviews_spec.rb+++ b/spec/models/reports/top_countries_by_browser_pageviews_spec.rb@@ -5,33 +5,29 @@ describe Reports::TopCountriesByBrowserPageviews dolet(:start_date) { 7.days.ago.to_date }let(:end_date) { Date.today }+ let(:report) do+ BrowserPageviewCountryDailyRollup.aggregate(start_date: start_date, end_date: end_date)+ BrowserPageviewEvent.delete_all+ Report.find("top_countries_by_browser_pageviews", start_date: start_date, end_date: end_date)+ end+it "ranks countries by event count and computes percent of total browser pageviews" do3.times { Fabricate(:browser_pageview_event, country_code: "US") }1.times { Fabricate(:browser_pageview_event, country_code: "GB") }- report =- Report.find(- "top_countries_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )- expect(report.data.map { |row| row[:country_code] }).to eq(%w[US GB])- expect(report.data.first[:count]).to eq(3)- expect(report.data.first[:percent]).to eq(75)+ data = report.data+ expect(data.map { |row| row[:country_code] }).to eq(%w[US GB])+ expect(data.first[:count]).to eq(3)+ expect(data.first[:percent]).to eq(75)endit "excludes NULL country_code from numerator but includes in denominator" do2.times { Fabricate(:browser_pageview_event, country_code: "US") }2.times { Fabricate(:browser_pageview_event, country_code: nil) }- report =- Report.find(- "top_countries_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )- expect(report.data.map { |row| row[:country_code] }).to eq(%w[US])- expect(report.data.first[:percent]).to eq(50)+ data = report.data+ expect(data.map { |row| row[:country_code] }).to eq(%w[US])+ expect(data.first[:percent]).to eq(50)endit "excludes MaxMind reserved country codes" do@@ -40,12 +36,6 @@ describe Reports::TopCountriesByBrowserPageviews doendFabricate(:browser_pageview_event, country_code: "US")- report =- Report.find(- "top_countries_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )expect(report.data.map { |row| row[:country_code] }).to eq(%w[US])end@@ -56,24 +46,13 @@ describe Reports::TopCountriesByBrowserPageviews doFabricate(:browser_pageview_event, country_code: "US") # anonymous, ignoredFabricate(:browser_pageview_event, country_code: "GB", user_id: user.id)- report =- Report.find(- "top_countries_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )- expect(report.data.map { |row| row[:country_code] }).to contain_exactly("US", "GB")- expect(report.data.first[:count]).to eq(1)- expect(report.data.first[:percent]).to eq(50)+ data = report.data+ expect(data.map { |row| row[:country_code] }).to contain_exactly("US", "GB")+ expect(data.first[:count]).to eq(1)+ expect(data.first[:percent]).to eq(50)endit "returns empty data when no events exist in range" do- report =- Report.find(- "top_countries_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )expect(report.data).to eq([])end@@ -83,12 +62,6 @@ describe Reports::TopCountriesByBrowserPageviews doFabricate(:browser_pageview_event, country_code: "US", created_at: end_of_day)Fabricate(:browser_pageview_event, country_code: "US", created_at: next_day) # out of range- report =- Report.find(- "top_countries_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )expect(report.data.first[:count]).to eq(1)end@@ -97,14 +70,16 @@ describe Reports::TopCountriesByBrowserPageviews do(idx + 1).times { Fabricate(:browser_pageview_event, country_code: code) }end- report =+ BrowserPageviewCountryDailyRollup.aggregate(start_date: start_date, end_date: end_date)+ BrowserPageviewEvent.delete_all+ limited =Report.find("top_countries_by_browser_pageviews",start_date: start_date,end_date: end_date,limit: 3,)- expect(report.data.size).to eq(3)+ expect(limited.data.size).to eq(3)endendenddiff --git a/spec/models/reports/top_referrers_by_browser_pageviews_spec.rb b/spec/models/reports/top_referrers_by_browser_pageviews_spec.rbindex f45acfdf7c2..96f9883fe22 100644--- a/spec/models/reports/top_referrers_by_browser_pageviews_spec.rb+++ b/spec/models/reports/top_referrers_by_browser_pageviews_spec.rb@@ -7,36 +7,32 @@ describe Reports::TopReferrersByBrowserPageviews dobefore { Discourse.stubs(:current_hostname).returns("forum.example.com") }+ let(:report) do+ BrowserPageviewReferrerDailyRollup.aggregate(start_date: start_date, end_date: end_date)+ BrowserPageviewEvent.delete_all+ Report.find("top_referrers_by_browser_pageviews", start_date: start_date, end_date: end_date)+ end+it "ranks referrers by event count and computes percent of total browser pageviews" do3.times doFabricate(:browser_pageview_event, normalized_referrer: "news.ycombinator.com/item?id=1")end1.times { Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com/r/discourse") }- report =- Report.find(- "top_referrers_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )- expect(report.data.map { |row| row[:normalized_referrer] }).to eq(+ data = report.data+ expect(data.map { |row| row[:normalized_referrer] }).to eq(%w[news.ycombinator.com/item?id=1 reddit.com/r/discourse],)- expect(report.data.first[:percent]).to eq(75)+ expect(data.first[:percent]).to eq(75)endit "excludes NULL normalized_referrer from numerator but includes in denominator" do2.times { Fabricate(:browser_pageview_event, normalized_referrer: "google.com") }2.times { Fabricate(:browser_pageview_event, normalized_referrer: nil) }- report =- Report.find(- "top_referrers_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )- expect(report.data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])- expect(report.data.first[:percent]).to eq(50)+ data = report.data+ expect(data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])+ expect(data.first[:percent]).to eq(50)endit "excludes same-host bare, path-prefixed, and query-prefixed referrers from numerator and denominator" do@@ -45,14 +41,9 @@ describe Reports::TopReferrersByBrowserPageviews doFabricate(:browser_pageview_event, normalized_referrer: "forum.example.com?ref=email")Fabricate(:browser_pageview_event, normalized_referrer: "google.com")- report =- Report.find(- "top_referrers_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )- expect(report.data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])- expect(report.data.first[:percent]).to eq(100)+ data = report.data+ expect(data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])+ expect(data.first[:percent]).to eq(100)endit "includes direct (no-referrer) pageviews in the denominator alongside external referrers" do@@ -60,24 +51,14 @@ describe Reports::TopReferrersByBrowserPageviews do2.times { Fabricate(:browser_pageview_event, normalized_referrer: nil) }Fabricate(:browser_pageview_event, normalized_referrer: "forum.example.com/t/topic/1")- report =- Report.find(- "top_referrers_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )- expect(report.data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])- expect(report.data.first[:percent]).to eq(50)+ data = report.data+ expect(data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])+ expect(data.first[:percent]).to eq(50)endit "does not exclude hosts that merely share a prefix with current_hostname" doFabricate(:browser_pageview_event, normalized_referrer: "evilforum.example.com/x")- report =- Report.find(- "top_referrers_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )+expect(report.data.map { |row| row[:normalized_referrer] }).to eq(["evilforum.example.com/x"])end@@ -86,12 +67,6 @@ describe Reports::TopReferrersByBrowserPageviews doFabricate(:browser_pageview_event, normalized_referrer: "forum.example.com")Fabricate(:browser_pageview_event, normalized_referrer: "google.com")- report =- Report.find(- "top_referrers_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )expect(report.data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])end@@ -100,12 +75,6 @@ describe Reports::TopReferrersByBrowserPageviews doFabricate(:browser_pageview_event, normalized_referrer: "xn--mnchen-3ya.de/blog")Fabricate(:browser_pageview_event, normalized_referrer: "google.com")- report =- Report.find(- "top_referrers_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )expect(report.data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])end@@ -116,12 +85,6 @@ describe Reports::TopReferrersByBrowserPageviews doFabricate(:browser_pageview_event, normalized_referrer: "my-blog.example.com/x")Fabricate(:browser_pageview_event, normalized_referrer: "google.com")- report =- Report.find(- "top_referrers_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )expect(report.data.map { |row| row[:normalized_referrer] }).to contain_exactly("my-blog.example.com/x","google.com",@@ -135,23 +98,11 @@ describe Reports::TopReferrersByBrowserPageviews doFabricate(:browser_pageview_event, normalized_referrer: "google.com") # anonymous, ignoredFabricate(:browser_pageview_event, normalized_referrer: "reddit.com", user_id: user.id)- report =- Report.find(- "top_referrers_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )expect(report.data.first[:count]).to eq(1)endit "returns empty data when no qualifying events exist" doFabricate(:browser_pageview_event, normalized_referrer: nil)- report =- Report.find(- "top_referrers_by_browser_pageviews",- start_date: start_date,- end_date: end_date,- )expect(report.data).to eq([])end@@ -162,14 +113,16 @@ describe Reports::TopReferrersByBrowserPageviews doendend- report =+ BrowserPageviewReferrerDailyRollup.aggregate(start_date: start_date, end_date: end_date)+ BrowserPageviewEvent.delete_all+ limited =Report.find("top_referrers_by_browser_pageviews",start_date: start_date,end_date: end_date,limit: 3,)- expect(report.data.size).to eq(3)+ expect(limited.data.size).to eq(3)endendenddiff --git a/spec/services/admin_dashboard_site_traffic_spec.rb b/spec/services/admin_dashboard_site_traffic_spec.rbindex 8d45d24bf79..78c2852fed6 100644--- a/spec/services/admin_dashboard_site_traffic_spec.rb+++ b/spec/services/admin_dashboard_site_traffic_spec.rb@@ -529,6 +529,12 @@ RSpec.describe AdminDashboardSiteTraffic docontext "for top countries and top referrers" dobefore { SiteSetting.persist_browser_pageview_events = true }+ def aggregate_rollups+ range = { start_date: 1.year.ago.to_date, end_date: Date.current }+ BrowserPageviewCountryDailyRollup.aggregate(**range)+ BrowserPageviewReferrerDailyRollup.aggregate(**range)+ end+it "omits top_countries and top_referrers when persist_browser_pageview_events is disabled" doSiteSetting.persist_browser_pageview_events = false@@ -541,6 +547,7 @@ RSpec.describe AdminDashboardSiteTraffic do6.times doFabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")end+ aggregate_rollupsresult = described_class.build(start_date: nil, end_date: nil)@@ -569,10 +576,13 @@ RSpec.describe AdminDashboardSiteTraffic do4.times doFabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")end+ aggregate_rollupsfirst = described_class.build(start_date: nil, end_date: nil)expect(first[:top_countries][:rows].first[:country_code]).to eq("US")+ BrowserPageviewCountryDailyRollup.delete_all+ BrowserPageviewReferrerDailyRollup.delete_allBrowserPageviewEvent.delete_allsecond = described_class.build(start_date: nil, end_date: nil)@@ -584,6 +594,7 @@ RSpec.describe AdminDashboardSiteTraffic do4.times doFabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")end+ aggregate_rollupsSiteSetting.login_required = falsefirst = described_class.build(start_date: nil, end_date: nil)@@ -597,6 +608,7 @@ RSpec.describe AdminDashboardSiteTraffic doit "invalidates the cached payload when current_hostname changes" doDiscourse.stubs(:current_hostname).returns("forum-a.example.com")Fabricate(:browser_pageview_event, normalized_referrer: "forum-b.example.com/path")+ aggregate_rollupsfirst = described_class.build(start_date: nil, end_date: nil)expect(first[:top_referrers][:rows].first[:normalized_referrer]).to eq(@@ -643,6 +655,7 @@ RSpec.describe AdminDashboardSiteTraffic doallow(Report).to receive(:find).and_call_originalFabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")+ aggregate_rollupssecond = described_class.build(start_date: nil, end_date: nil)expect(second[:top_countries][:rows].first[:country_code]).to eq("US")diff --git a/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb b/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rbindex b58792f46ca..c2b47774026 100644--- a/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb+++ b/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb@@ -178,6 +178,15 @@ describe "Admin Dashboard Redesign | Site Traffic section" do)end+ BrowserPageviewCountryDailyRollup.aggregate(+ start_date: "2026-05-01".to_date,+ end_date: "2026-05-14".to_date,+ )+ BrowserPageviewReferrerDailyRollup.aggregate(+ start_date: "2026-05-01".to_date,+ end_date: "2026-05-14".to_date,+ )+dashboard.visittraffic = dashboard.site_traffic