Skip to content

A platform for community discussion. Free, open, simple.

PERF: Daily rollups for top countries and top referrers reports (#40285)

Alan Guo Xiang Tan2cc3ac34e73b
diff --git a/app/jobs/scheduled/aggregate_browser_pageview_daily_rollups.rb b/app/jobs/scheduled/aggregate_browser_pageview_daily_rollups.rb
new file mode 100644
index 00000000000..cc34d0bd759
--- /dev/null
+++ b/app/jobs/scheduled/aggregate_browser_pageview_daily_rollups.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Jobs
+ class AggregateBrowserPageviewDailyRollups < ::Jobs::Scheduled
+ every 30.minutes
+
+ LOCK_KEY = "aggregate_browser_pageview_daily_rollups"
+
+ def execute(args)
+ return if !SiteSetting.persist_browser_pageview_events
+
+ start_date, end_date = aggregation_window
+ return if start_date.nil?
+
+ DistributedMutex.synchronize(LOCK_KEY, validity: 10.minutes) do
+ BrowserPageviewCountryDailyRollup.aggregate(start_date: start_date, end_date: end_date)
+ BrowserPageviewReferrerDailyRollup.aggregate(start_date: start_date, end_date: end_date)
+ end
+
+ Report.clear_cache("top_countries_by_browser_pageviews")
+ Report.clear_cache("top_referrers_by_browser_pageviews")
+ end
+
+ private
+
+ def aggregation_window
+ end_date = Time.zone.today
+
+ if BrowserPageviewCountryDailyRollup.none? && BrowserPageviewReferrerDailyRollup.none?
+ earliest_event_date = BrowserPageviewEvent.minimum(:created_at)&.to_date
+ [earliest_event_date, end_date]
+ else
+ [1.day.ago.to_date, end_date]
+ end
+ end
+ end
+end
diff --git a/app/models/browser_pageview_country_daily_rollup.rb b/app/models/browser_pageview_country_daily_rollup.rb
new file mode 100644
index 00000000000..60cd54a4e34
--- /dev/null
+++ b/app/models/browser_pageview_country_daily_rollup.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class BrowserPageviewCountryDailyRollup < ActiveRecord::Base
+ def self.aggregate(start_date:, end_date:)
+ DB.exec(<<~SQL, start_date: start_date.to_date, end_date: end_date.to_date + 1)
+ INSERT INTO browser_pageview_country_daily_rollups (date, country_code, count, logged_in_count)
+ SELECT
+ created_at::date AS date,
+ country_code,
+ COUNT(*) AS count,
+ COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS logged_in_count
+ FROM browser_pageview_events
+ WHERE created_at >= :start_date
+ AND created_at < :end_date
+ GROUP BY date, country_code
+ ON CONFLICT (date, country_code) DO UPDATE
+ SET count = EXCLUDED.count,
+ logged_in_count = EXCLUDED.logged_in_count
+ SQL
+ end
+end
+
+# == Schema Information
+#
+# Table name: browser_pageview_country_daily_rollups
+#
+# id :bigint not null, primary key
+# count :bigint not null
+# country_code :string(2)
+# date :date not null
+# logged_in_count :bigint not null
+#
+# Indexes
+#
+# idx_bpcd_rollups_date_country_unique (date,country_code) UNIQUE NULLS NOT DISTINCT
+#
diff --git a/app/models/browser_pageview_referrer_daily_rollup.rb b/app/models/browser_pageview_referrer_daily_rollup.rb
new file mode 100644
index 00000000000..e210d23172d
--- /dev/null
+++ b/app/models/browser_pageview_referrer_daily_rollup.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class BrowserPageviewReferrerDailyRollup < ActiveRecord::Base
+ def self.aggregate(start_date:, end_date:)
+ DB.exec(<<~SQL, start_date: start_date.to_date, end_date: end_date.to_date + 1)
+ INSERT INTO browser_pageview_referrer_daily_rollups (date, normalized_referrer, count, logged_in_count)
+ SELECT
+ created_at::date AS date,
+ normalized_referrer,
+ COUNT(*) AS count,
+ COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS logged_in_count
+ FROM browser_pageview_events
+ WHERE created_at >= :start_date
+ AND created_at < :end_date
+ GROUP BY date, normalized_referrer
+ ON CONFLICT (date, normalized_referrer) DO UPDATE
+ SET count = EXCLUDED.count,
+ logged_in_count = EXCLUDED.logged_in_count
+ SQL
+ end
+end
+
+# == Schema Information
+#
+# Table name: browser_pageview_referrer_daily_rollups
+#
+# id :bigint not null, primary key
+# count :bigint not null
+# date :date not null
+# logged_in_count :bigint not null
+# normalized_referrer :string(2000)
+#
+# Indexes
+#
+# idx_bprd_rollups_date_referrer_unique (date,normalized_referrer) UNIQUE NULLS NOT DISTINCT
+#
diff --git a/app/models/concerns/reports/top_countries_by_browser_pageviews.rb b/app/models/concerns/reports/top_countries_by_browser_pageviews.rb
index 0d7cfb6c1f4..75dfc88892e 100644
--- a/app/models/concerns/reports/top_countries_by_browser_pageviews.rb
+++ b/app/models/concerns/reports/top_countries_by_browser_pageviews.rb
@@ -22,20 +22,20 @@ module Reports::TopCountriesByBrowserPageviews
},
]
- user_filter_sql = SiteSetting.login_required ? "AND user_id IS NOT NULL" : ""
+ count_expr = SiteSetting.login_required ? "logged_in_count" : "count"
end_date_exclusive = report.end_date.to_date + 1
sql = <<~SQL
WITH ranked AS (
SELECT
country_code,
- COUNT(*) AS count,
- SUM(COUNT(*)) OVER () AS total
- FROM browser_pageview_events
- WHERE created_at >= :start_date
- AND created_at < :end_date_exclusive
- #{user_filter_sql}
+ SUM(#{count_expr}) AS count,
+ SUM(SUM(#{count_expr})) OVER () AS total
+ FROM browser_pageview_country_daily_rollups
+ WHERE date >= :start_date
+ AND date < :end_date_exclusive
GROUP BY country_code
+ HAVING SUM(#{count_expr}) > 0
)
SELECT country_code, count,
CASE WHEN total = 0 THEN 0
diff --git a/app/models/concerns/reports/top_referrers_by_browser_pageviews.rb b/app/models/concerns/reports/top_referrers_by_browser_pageviews.rb
index b1858fe4105..6eec87d775e 100644
--- a/app/models/concerns/reports/top_referrers_by_browser_pageviews.rb
+++ b/app/models/concerns/reports/top_referrers_by_browser_pageviews.rb
@@ -20,7 +20,7 @@ module Reports::TopReferrersByBrowserPageviews
},
]
- user_filter_sql = SiteSetting.login_required ? "AND user_id IS NOT NULL" : ""
+ count_expr = SiteSetting.login_required ? "logged_in_count" : "count"
end_date_exclusive = report.end_date.to_date + 1
host = BrowserPageviewReferrerInspector.normalize_host(Discourse.current_hostname)
@@ -30,12 +30,11 @@ module Reports::TopReferrersByBrowserPageviews
WITH ranked AS (
SELECT
normalized_referrer,
- COUNT(*) AS count,
- SUM(COUNT(*)) OVER () AS total
- FROM browser_pageview_events
- WHERE created_at >= :start_date
- AND created_at < :end_date_exclusive
- #{user_filter_sql}
+ SUM(#{count_expr}) AS count,
+ SUM(SUM(#{count_expr})) OVER () AS total
+ FROM browser_pageview_referrer_daily_rollups
+ WHERE date >= :start_date
+ AND date < :end_date_exclusive
AND (
normalized_referrer IS NULL
OR (
@@ -45,6 +44,7 @@ module Reports::TopReferrersByBrowserPageviews
)
)
GROUP BY normalized_referrer
+ HAVING SUM(#{count_expr}) > 0
)
SELECT normalized_referrer, count,
CASE WHEN total = 0 THEN 0
diff --git a/db/migrate/20260525105006_create_browser_pageview_country_daily_rollups.rb b/db/migrate/20260525105006_create_browser_pageview_country_daily_rollups.rb
new file mode 100644
index 00000000000..f7c524d6aeb
--- /dev/null
+++ b/db/migrate/20260525105006_create_browser_pageview_country_daily_rollups.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+class CreateBrowserPageviewCountryDailyRollups < ActiveRecord::Migration[8.0]
+ def change
+ create_table :browser_pageview_country_daily_rollups do |t|
+ t.date :date, null: false
+ t.string :country_code, limit: 2
+ t.bigint :count, null: false
+ t.bigint :logged_in_count, null: false
+ end
+
+ add_index :browser_pageview_country_daily_rollups,
+ %i[date country_code],
+ unique: true,
+ nulls_not_distinct: true,
+ name: "idx_bpcd_rollups_date_country_unique"
+ end
+end
diff --git a/db/migrate/20260525105009_create_browser_pageview_referrer_daily_rollups.rb b/db/migrate/20260525105009_create_browser_pageview_referrer_daily_rollups.rb
new file mode 100644
index 00000000000..053f11abca7
--- /dev/null
+++ b/db/migrate/20260525105009_create_browser_pageview_referrer_daily_rollups.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+class CreateBrowserPageviewReferrerDailyRollups < ActiveRecord::Migration[8.0]
+ def change
+ create_table :browser_pageview_referrer_daily_rollups do |t|
+ t.date :date, null: false
+ t.string :normalized_referrer, limit: 2000
+ t.bigint :count, null: false
+ t.bigint :logged_in_count, null: false
+ end
+
+ add_index :browser_pageview_referrer_daily_rollups,
+ %i[date normalized_referrer],
+ unique: true,
+ nulls_not_distinct: true,
+ name: "idx_bprd_rollups_date_referrer_unique"
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index 88a26a63a9f..0f6d2118587 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1833,6 +1833,38 @@ CREATE SEQUENCE public.bookmarks_id_seq
ALTER SEQUENCE public.bookmarks_id_seq OWNED BY public.bookmarks.id;
+--
+-- Name: browser_pageview_country_daily_rollups; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.browser_pageview_country_daily_rollups (
+ id bigint NOT NULL,
+ date date NOT NULL,
+ country_code character varying(2),
+ count bigint NOT NULL,
+ logged_in_count bigint NOT NULL
+);
+
+
+--
+-- Name: browser_pageview_country_daily_rollups_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.browser_pageview_country_daily_rollups_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+--
+-- Name: browser_pageview_country_daily_rollups_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.browser_pageview_country_daily_rollups_id_seq OWNED BY public.browser_pageview_country_daily_rollups.id;
+
+
--
-- Name: browser_pageview_event_scores; Type: TABLE; Schema: public; Owner: -
--
@@ -1908,6 +1940,38 @@ CREATE SEQUENCE public.browser_pageview_events_id_seq
ALTER SEQUENCE public.browser_pageview_events_id_seq OWNED BY public.browser_pageview_events.id;
+--
+-- Name: browser_pageview_referrer_daily_rollups; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.browser_pageview_referrer_daily_rollups (
+ id bigint NOT NULL,
+ date date NOT NULL,
+ normalized_referrer character varying(2000),
+ count bigint NOT NULL,
+ logged_in_count bigint NOT NULL
+);
+
+
+--
+-- Name: browser_pageview_referrer_daily_rollups_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.browser_pageview_referrer_daily_rollups_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+--
+-- Name: browser_pageview_referrer_daily_rollups_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.browser_pageview_referrer_daily_rollups_id_seq OWNED BY public.browser_pageview_referrer_daily_rollups.id;
+
+
--
-- Name: calendar_events; Type: TABLE; Schema: public; Owner: -
--
@@ -11592,6 +11656,13 @@ ALTER TABLE ONLY public.badges ALTER COLUMN id SET DEFAULT nextval('public.badge
ALTER TABLE ONLY public.bookmarks ALTER COLUMN id SET DEFAULT nextval('public.bookmarks_id_seq'::regclass);
+--
+-- Name: browser_pageview_country_daily_rollups id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.browser_pageview_country_daily_rollups ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_country_daily_rollups_id_seq'::regclass);
+
+
--
-- Name: browser_pageview_event_scores id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -11606,6 +11677,13 @@ ALTER TABLE ONLY public.browser_pageview_event_scores ALTER COLUMN id SET DEFAUL
ALTER TABLE ONLY public.browser_pageview_events ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_events_id_seq'::regclass);
+--
+-- Name: browser_pageview_referrer_daily_rollups id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.browser_pageview_referrer_daily_rollups ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_referrer_daily_rollups_id_seq'::regclass);
+
+
--
-- Name: calendar_events id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -13671,6 +13749,14 @@ ALTER TABLE ONLY public.bookmarks
ADD CONSTRAINT bookmarks_pkey PRIMARY KEY (id);
+--
+-- Name: browser_pageview_country_daily_rollups browser_pageview_country_daily_rollups_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.browser_pageview_country_daily_rollups
+ ADD CONSTRAINT browser_pageview_country_daily_rollups_pkey PRIMARY KEY (id);
+
+
--
-- Name: browser_pageview_event_scores browser_pageview_event_scores_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -13687,6 +13773,14 @@ ALTER TABLE ONLY public.browser_pageview_events
ADD CONSTRAINT browser_pageview_events_pkey PRIMARY KEY (id);
+--
+-- Name: browser_pageview_referrer_daily_rollups browser_pageview_referrer_daily_rollups_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.browser_pageview_referrer_daily_rollups
+ ADD CONSTRAINT browser_pageview_referrer_daily_rollups_pkey PRIMARY KEY (id);
+
+
--
-- Name: calendar_events calendar_events_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -15889,6 +15983,13 @@ CREATE UNIQUE INDEX idx_ai_bot_conversation_stars_user_topic ON public.discourse
CREATE UNIQUE INDEX idx_bookmarks_user_polymorphic_unique ON public.bookmarks USING btree (user_id, bookmarkable_type, bookmarkable_id);
+--
+-- Name: idx_bpcd_rollups_date_country_unique; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX idx_bpcd_rollups_date_country_unique ON public.browser_pageview_country_daily_rollups USING btree (date, country_code) NULLS NOT DISTINCT;
+
+
--
-- Name: idx_bpe_created_at_country_code; Type: INDEX; Schema: public; Owner: -
--
@@ -15917,6 +16018,13 @@ CREATE INDEX idx_bpe_ip_ua_created_at ON public.browser_pageview_events USING bt
CREATE INDEX idx_bpe_session_created_at ON public.browser_pageview_events USING btree (session_id, created_at);
+--
+-- Name: idx_bprd_rollups_date_referrer_unique; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX idx_bprd_rollups_date_referrer_unique ON public.browser_pageview_referrer_daily_rollups USING btree (date, normalized_referrer) NULLS NOT DISTINCT;
+
+
--
-- Name: idx_category_posting_review_groups_unique; Type: INDEX; Schema: public; Owner: -
--
@@ -20938,6 +21046,8 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groups
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
+('20260525105009'),
+('20260525105006'),
('20260522043337'),
('20260520090937'),
('20260518104900'),
diff --git a/spec/jobs/aggregate_browser_pageview_daily_rollups_spec.rb b/spec/jobs/aggregate_browser_pageview_daily_rollups_spec.rb
new file mode 100644
index 00000000000..edbc9cbd967
--- /dev/null
+++ b/spec/jobs/aggregate_browser_pageview_daily_rollups_spec.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+RSpec.describe Jobs::AggregateBrowserPageviewDailyRollups do
+ before { Discourse.stubs(:current_hostname).returns("forum.example.com") }
+
+ it "does nothing when persist_browser_pageview_events is disabled" do
+ SiteSetting.persist_browser_pageview_events = false
+ Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")
+
+ expect { described_class.new.execute({}) }.not_to change {
+ BrowserPageviewCountryDailyRollup.count + BrowserPageviewReferrerDailyRollup.count
+ }
+ end
+
+ it "aggregates both country and referrer rollups from recent pageview events" do
+ SiteSetting.persist_browser_pageview_events = true
+ Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")
+
+ described_class.new.execute({})
+
+ expect(BrowserPageviewCountryDailyRollup.where(country_code: "US").sum(:count)).to eq(1)
+ expect(
+ BrowserPageviewReferrerDailyRollup.where(normalized_referrer: "google.com").sum(:count),
+ ).to eq(1)
+ end
+
+ it "backfills from the earliest event date on the first run when rollups are empty" do
+ SiteSetting.persist_browser_pageview_events = true
+ Fabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)
+ Fabricate(:browser_pageview_event, country_code: "GB", created_at: 5.days.ago)
+
+ described_class.new.execute({})
+
+ expect(BrowserPageviewCountryDailyRollup.pluck(:country_code)).to contain_exactly("US", "GB")
+ end
+
+ it "clears the report cache after running so stale empty results do not linger" do
+ SiteSetting.persist_browser_pageview_events = true
+ Fabricate(:browser_pageview_event, country_code: "US")
+ Report.expects(:clear_cache).with("top_countries_by_browser_pageviews").once
+ Report.expects(:clear_cache).with("top_referrers_by_browser_pageviews").once
+
+ described_class.new.execute({})
+ end
+
+ it "only aggregates yesterday and today once historical rollups are populated" do
+ SiteSetting.persist_browser_pageview_events = true
+ Fabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)
+ described_class.new.execute({}) # first run backfills everything
+
+ Fabricate(:browser_pageview_event, country_code: "GB", created_at: 60.days.ago) # late old event
+ Fabricate(:browser_pageview_event, country_code: "FR") # today
+ described_class.new.execute({}) # second run only does yesterday + today
+
+ expect(BrowserPageviewCountryDailyRollup.pluck(:country_code)).to contain_exactly("US", "FR")
+ end
+end
diff --git a/spec/models/browser_pageview_country_daily_rollup_spec.rb b/spec/models/browser_pageview_country_daily_rollup_spec.rb
new file mode 100644
index 00000000000..0564a6dde1c
--- /dev/null
+++ b/spec/models/browser_pageview_country_daily_rollup_spec.rb
@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+RSpec.describe BrowserPageviewCountryDailyRollup do
+ describe ".aggregate" do
+ let(:start_date) { 3.days.ago.to_date }
+ let(:end_date) { Date.current }
+
+ it "groups events by date and country_code into one row per combination" do
+ yesterday = 1.day.ago
+ today = Time.current
+ Fabricate(:browser_pageview_event, country_code: "US", created_at: yesterday)
+ Fabricate(:browser_pageview_event, country_code: "US", created_at: today)
+ Fabricate(:browser_pageview_event, country_code: "GB", created_at: today)
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ rollups = described_class.order(:date, :country_code).pluck(:date, :country_code, :count)
+ expect(rollups).to contain_exactly(
+ [yesterday.to_date, "US", 1],
+ [today.to_date, "US", 1],
+ [today.to_date, "GB", 1],
+ )
+ end
+
+ it "splits total count from logged-in count" do
+ user = Fabricate(:user)
+ Fabricate(:browser_pageview_event, country_code: "US", user_id: user.id)
+ Fabricate(:browser_pageview_event, country_code: "US")
+ Fabricate(:browser_pageview_event, country_code: "US")
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ row = described_class.first
+ expect(row.count).to eq(3)
+ expect(row.logged_in_count).to eq(1)
+ end
+
+ it "stores rows for NULL country_code (no GeoIP match) without duplication on re-run" do
+ Fabricate(:browser_pageview_event, country_code: nil)
+ Fabricate(:browser_pageview_event, country_code: nil)
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ null_rows = described_class.where(country_code: nil)
+ expect(null_rows.count).to eq(1)
+ expect(null_rows.first.count).to eq(2)
+ end
+
+ it "is idempotent — running twice produces the same totals" do
+ Fabricate(:browser_pageview_event, country_code: "US")
+ Fabricate(:browser_pageview_event, country_code: "US")
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ expect(described_class.where(country_code: "US").sum(:count)).to eq(2)
+ end
+
+ it "updates existing rollup rows when re-aggregating with new events" do
+ Fabricate(:browser_pageview_event, country_code: "US")
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ Fabricate(:browser_pageview_event, country_code: "US")
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ expect(described_class.where(country_code: "US").sum(:count)).to eq(2)
+ end
+
+ it "only aggregates events within the requested date range" do
+ Fabricate(:browser_pageview_event, country_code: "US", created_at: 10.days.ago)
+ Fabricate(:browser_pageview_event, country_code: "US", created_at: 1.day.ago)
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ expect(described_class.sum(:count)).to eq(1)
+ end
+
+ it "is a no-op when no events exist in the range" do
+ expect {
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+ }.not_to change { described_class.count }
+ end
+ end
+end
diff --git a/spec/models/browser_pageview_referrer_daily_rollup_spec.rb b/spec/models/browser_pageview_referrer_daily_rollup_spec.rb
new file mode 100644
index 00000000000..c1d6ae6aee1
--- /dev/null
+++ b/spec/models/browser_pageview_referrer_daily_rollup_spec.rb
@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+RSpec.describe BrowserPageviewReferrerDailyRollup do
+ describe ".aggregate" do
+ let(:start_date) { 3.days.ago.to_date }
+ let(:end_date) { Date.current }
+
+ it "groups events by date and normalized_referrer into one row per combination" do
+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com")
+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com")
+ Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com")
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ rollups = described_class.order(:normalized_referrer).pluck(:normalized_referrer, :count)
+ expect(rollups).to contain_exactly(["google.com", 2], ["reddit.com", 1])
+ end
+
+ it "splits total count from logged-in count" do
+ user = Fabricate(:user)
+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com", user_id: user.id)
+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com")
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ row = described_class.find_by(normalized_referrer: "google.com")
+ expect(row.count).to eq(2)
+ expect(row.logged_in_count).to eq(1)
+ end
+
+ it "stores NULL-referrer rows (direct visits)" do
+ Fabricate(:browser_pageview_event, normalized_referrer: nil)
+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com")
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ expect(described_class.where(normalized_referrer: nil).first.count).to eq(1)
+ end
+
+ it "stores internal-referrer rows so the data remains recoverable on hostname or rule changes" do
+ Fabricate(:browser_pageview_event, normalized_referrer: "forum.example.com")
+ Fabricate(:browser_pageview_event, normalized_referrer: "forum.example.com/t/topic/1")
+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com")
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ expect(described_class.pluck(:normalized_referrer)).to contain_exactly(
+ "forum.example.com",
+ "forum.example.com/t/topic/1",
+ "google.com",
+ )
+ end
+
+ it "is idempotent — running twice produces the same totals" do
+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com")
+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com")
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ expect(described_class.where(normalized_referrer: "google.com").sum(:count)).to eq(2)
+ end
+
+ it "is idempotent for NULL referrer rows" do
+ Fabricate(:browser_pageview_event, normalized_referrer: nil)
+ Fabricate(:browser_pageview_event, normalized_referrer: nil)
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ null_rows = described_class.where(normalized_referrer: nil)
+ expect(null_rows.count).to eq(1)
+ expect(null_rows.first.count).to eq(2)
+ end
+
+ it "only aggregates events within the requested date range" do
+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com", created_at: 10.days.ago)
+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com", created_at: 1.day.ago)
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ expect(described_class.sum(:count)).to eq(1)
+ end
+ end
+end
diff --git a/spec/models/reports/top_countries_by_browser_pageviews_spec.rb b/spec/models/reports/top_countries_by_browser_pageviews_spec.rb
index 13b7b967842..c13daceb7cd 100644
--- a/spec/models/reports/top_countries_by_browser_pageviews_spec.rb
+++ b/spec/models/reports/top_countries_by_browser_pageviews_spec.rb
@@ -5,33 +5,29 @@ describe Reports::TopCountriesByBrowserPageviews do
let(:start_date) { 7.days.ago.to_date }
let(:end_date) { Date.today }
+ let(:report) do
+ BrowserPageviewCountryDailyRollup.aggregate(start_date: start_date, end_date: end_date)
+ BrowserPageviewEvent.delete_all
+ Report.find("top_countries_by_browser_pageviews", start_date: start_date, end_date: end_date)
+ end
+
it "ranks countries by event count and computes percent of total browser pageviews" do
3.times { Fabricate(:browser_pageview_event, country_code: "US") }
1.times { Fabricate(:browser_pageview_event, country_code: "GB") }
- report =
- Report.find(
- "top_countries_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
- expect(report.data.map { |row| row[:country_code] }).to eq(%w[US GB])
- expect(report.data.first[:count]).to eq(3)
- expect(report.data.first[:percent]).to eq(75)
+ data = report.data
+ expect(data.map { |row| row[:country_code] }).to eq(%w[US GB])
+ expect(data.first[:count]).to eq(3)
+ expect(data.first[:percent]).to eq(75)
end
it "excludes NULL country_code from numerator but includes in denominator" do
2.times { Fabricate(:browser_pageview_event, country_code: "US") }
2.times { Fabricate(:browser_pageview_event, country_code: nil) }
- report =
- Report.find(
- "top_countries_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
- expect(report.data.map { |row| row[:country_code] }).to eq(%w[US])
- expect(report.data.first[:percent]).to eq(50)
+ data = report.data
+ expect(data.map { |row| row[:country_code] }).to eq(%w[US])
+ expect(data.first[:percent]).to eq(50)
end
it "excludes MaxMind reserved country codes" do
@@ -40,12 +36,6 @@ describe Reports::TopCountriesByBrowserPageviews do
end
Fabricate(:browser_pageview_event, country_code: "US")
- report =
- Report.find(
- "top_countries_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
expect(report.data.map { |row| row[:country_code] }).to eq(%w[US])
end
@@ -56,24 +46,13 @@ describe Reports::TopCountriesByBrowserPageviews do
Fabricate(:browser_pageview_event, country_code: "US") # anonymous, ignored
Fabricate(:browser_pageview_event, country_code: "GB", user_id: user.id)
- report =
- Report.find(
- "top_countries_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
- expect(report.data.map { |row| row[:country_code] }).to contain_exactly("US", "GB")
- expect(report.data.first[:count]).to eq(1)
- expect(report.data.first[:percent]).to eq(50)
+ data = report.data
+ expect(data.map { |row| row[:country_code] }).to contain_exactly("US", "GB")
+ expect(data.first[:count]).to eq(1)
+ expect(data.first[:percent]).to eq(50)
end
it "returns empty data when no events exist in range" do
- report =
- Report.find(
- "top_countries_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
expect(report.data).to eq([])
end
@@ -83,12 +62,6 @@ describe Reports::TopCountriesByBrowserPageviews do
Fabricate(:browser_pageview_event, country_code: "US", created_at: end_of_day)
Fabricate(:browser_pageview_event, country_code: "US", created_at: next_day) # out of range
- report =
- Report.find(
- "top_countries_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
expect(report.data.first[:count]).to eq(1)
end
@@ -97,14 +70,16 @@ describe Reports::TopCountriesByBrowserPageviews do
(idx + 1).times { Fabricate(:browser_pageview_event, country_code: code) }
end
- report =
+ BrowserPageviewCountryDailyRollup.aggregate(start_date: start_date, end_date: end_date)
+ BrowserPageviewEvent.delete_all
+ limited =
Report.find(
"top_countries_by_browser_pageviews",
start_date: start_date,
end_date: end_date,
limit: 3,
)
- expect(report.data.size).to eq(3)
+ expect(limited.data.size).to eq(3)
end
end
end
diff --git a/spec/models/reports/top_referrers_by_browser_pageviews_spec.rb b/spec/models/reports/top_referrers_by_browser_pageviews_spec.rb
index f45acfdf7c2..96f9883fe22 100644
--- a/spec/models/reports/top_referrers_by_browser_pageviews_spec.rb
+++ b/spec/models/reports/top_referrers_by_browser_pageviews_spec.rb
@@ -7,36 +7,32 @@ describe Reports::TopReferrersByBrowserPageviews do
before { Discourse.stubs(:current_hostname).returns("forum.example.com") }
+ let(:report) do
+ BrowserPageviewReferrerDailyRollup.aggregate(start_date: start_date, end_date: end_date)
+ BrowserPageviewEvent.delete_all
+ Report.find("top_referrers_by_browser_pageviews", start_date: start_date, end_date: end_date)
+ end
+
it "ranks referrers by event count and computes percent of total browser pageviews" do
3.times do
Fabricate(:browser_pageview_event, normalized_referrer: "news.ycombinator.com/item?id=1")
end
1.times { Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com/r/discourse") }
- report =
- Report.find(
- "top_referrers_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
- expect(report.data.map { |row| row[:normalized_referrer] }).to eq(
+ data = report.data
+ expect(data.map { |row| row[:normalized_referrer] }).to eq(
%w[news.ycombinator.com/item?id=1 reddit.com/r/discourse],
)
- expect(report.data.first[:percent]).to eq(75)
+ expect(data.first[:percent]).to eq(75)
end
it "excludes NULL normalized_referrer from numerator but includes in denominator" do
2.times { Fabricate(:browser_pageview_event, normalized_referrer: "google.com") }
2.times { Fabricate(:browser_pageview_event, normalized_referrer: nil) }
- report =
- Report.find(
- "top_referrers_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
- expect(report.data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])
- expect(report.data.first[:percent]).to eq(50)
+ data = report.data
+ expect(data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])
+ expect(data.first[:percent]).to eq(50)
end
it "excludes same-host bare, path-prefixed, and query-prefixed referrers from numerator and denominator" do
@@ -45,14 +41,9 @@ describe Reports::TopReferrersByBrowserPageviews do
Fabricate(:browser_pageview_event, normalized_referrer: "forum.example.com?ref=email")
Fabricate(:browser_pageview_event, normalized_referrer: "google.com")
- report =
- Report.find(
- "top_referrers_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
- expect(report.data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])
- expect(report.data.first[:percent]).to eq(100)
+ data = report.data
+ expect(data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])
+ expect(data.first[:percent]).to eq(100)
end
it "includes direct (no-referrer) pageviews in the denominator alongside external referrers" do
@@ -60,24 +51,14 @@ describe Reports::TopReferrersByBrowserPageviews do
2.times { Fabricate(:browser_pageview_event, normalized_referrer: nil) }
Fabricate(:browser_pageview_event, normalized_referrer: "forum.example.com/t/topic/1")
- report =
- Report.find(
- "top_referrers_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
- expect(report.data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])
- expect(report.data.first[:percent]).to eq(50)
+ data = report.data
+ expect(data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])
+ expect(data.first[:percent]).to eq(50)
end
it "does not exclude hosts that merely share a prefix with current_hostname" do
Fabricate(:browser_pageview_event, normalized_referrer: "evilforum.example.com/x")
- report =
- Report.find(
- "top_referrers_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
+
expect(report.data.map { |row| row[:normalized_referrer] }).to eq(["evilforum.example.com/x"])
end
@@ -86,12 +67,6 @@ describe Reports::TopReferrersByBrowserPageviews do
Fabricate(:browser_pageview_event, normalized_referrer: "forum.example.com")
Fabricate(:browser_pageview_event, normalized_referrer: "google.com")
- report =
- Report.find(
- "top_referrers_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
expect(report.data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])
end
@@ -100,12 +75,6 @@ describe Reports::TopReferrersByBrowserPageviews do
Fabricate(:browser_pageview_event, normalized_referrer: "xn--mnchen-3ya.de/blog")
Fabricate(:browser_pageview_event, normalized_referrer: "google.com")
- report =
- Report.find(
- "top_referrers_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
expect(report.data.map { |row| row[:normalized_referrer] }).to eq(["google.com"])
end
@@ -116,12 +85,6 @@ describe Reports::TopReferrersByBrowserPageviews do
Fabricate(:browser_pageview_event, normalized_referrer: "my-blog.example.com/x")
Fabricate(:browser_pageview_event, normalized_referrer: "google.com")
- report =
- Report.find(
- "top_referrers_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
expect(report.data.map { |row| row[:normalized_referrer] }).to contain_exactly(
"my-blog.example.com/x",
"google.com",
@@ -135,23 +98,11 @@ describe Reports::TopReferrersByBrowserPageviews do
Fabricate(:browser_pageview_event, normalized_referrer: "google.com") # anonymous, ignored
Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com", user_id: user.id)
- report =
- Report.find(
- "top_referrers_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
expect(report.data.first[:count]).to eq(1)
end
it "returns empty data when no qualifying events exist" do
Fabricate(:browser_pageview_event, normalized_referrer: nil)
- report =
- Report.find(
- "top_referrers_by_browser_pageviews",
- start_date: start_date,
- end_date: end_date,
- )
expect(report.data).to eq([])
end
@@ -162,14 +113,16 @@ describe Reports::TopReferrersByBrowserPageviews do
end
end
- report =
+ BrowserPageviewReferrerDailyRollup.aggregate(start_date: start_date, end_date: end_date)
+ BrowserPageviewEvent.delete_all
+ limited =
Report.find(
"top_referrers_by_browser_pageviews",
start_date: start_date,
end_date: end_date,
limit: 3,
)
- expect(report.data.size).to eq(3)
+ expect(limited.data.size).to eq(3)
end
end
end
diff --git a/spec/services/admin_dashboard_site_traffic_spec.rb b/spec/services/admin_dashboard_site_traffic_spec.rb
index 8d45d24bf79..78c2852fed6 100644
--- a/spec/services/admin_dashboard_site_traffic_spec.rb
+++ b/spec/services/admin_dashboard_site_traffic_spec.rb
@@ -529,6 +529,12 @@ RSpec.describe AdminDashboardSiteTraffic do
context "for top countries and top referrers" do
before { SiteSetting.persist_browser_pageview_events = true }
+ def aggregate_rollups
+ range = { start_date: 1.year.ago.to_date, end_date: Date.current }
+ BrowserPageviewCountryDailyRollup.aggregate(**range)
+ BrowserPageviewReferrerDailyRollup.aggregate(**range)
+ end
+
it "omits top_countries and top_referrers when persist_browser_pageview_events is disabled" do
SiteSetting.persist_browser_pageview_events = false
@@ -541,6 +547,7 @@ RSpec.describe AdminDashboardSiteTraffic do
6.times do
Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")
end
+ aggregate_rollups
result = described_class.build(start_date: nil, end_date: nil)
@@ -569,10 +576,13 @@ RSpec.describe AdminDashboardSiteTraffic do
4.times do
Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")
end
+ aggregate_rollups
first = described_class.build(start_date: nil, end_date: nil)
expect(first[:top_countries][:rows].first[:country_code]).to eq("US")
+ BrowserPageviewCountryDailyRollup.delete_all
+ BrowserPageviewReferrerDailyRollup.delete_all
BrowserPageviewEvent.delete_all
second = described_class.build(start_date: nil, end_date: nil)
@@ -584,6 +594,7 @@ RSpec.describe AdminDashboardSiteTraffic do
4.times do
Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")
end
+ aggregate_rollups
SiteSetting.login_required = false
first = described_class.build(start_date: nil, end_date: nil)
@@ -597,6 +608,7 @@ RSpec.describe AdminDashboardSiteTraffic do
it "invalidates the cached payload when current_hostname changes" do
Discourse.stubs(:current_hostname).returns("forum-a.example.com")
Fabricate(:browser_pageview_event, normalized_referrer: "forum-b.example.com/path")
+ aggregate_rollups
first = described_class.build(start_date: nil, end_date: nil)
expect(first[:top_referrers][:rows].first[:normalized_referrer]).to eq(
@@ -643,6 +655,7 @@ RSpec.describe AdminDashboardSiteTraffic do
allow(Report).to receive(:find).and_call_original
Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")
+ aggregate_rollups
second = described_class.build(start_date: nil, end_date: nil)
expect(second[:top_countries][:rows].first[:country_code]).to eq("US")
diff --git a/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb b/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb
index b58792f46ca..c2b47774026 100644
--- a/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb
+++ b/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb
@@ -178,6 +178,15 @@ describe "Admin Dashboard Redesign | Site Traffic section" do
)
end
+ BrowserPageviewCountryDailyRollup.aggregate(
+ start_date: "2026-05-01".to_date,
+ end_date: "2026-05-14".to_date,
+ )
+ BrowserPageviewReferrerDailyRollup.aggregate(
+ start_date: "2026-05-01".to_date,
+ end_date: "2026-05-14".to_date,
+ )
+
dashboard.visit
traffic = dashboard.site_traffic