Skip to content

A platform for community discussion. Free, open, simple.

FIX: Backfill normalized referrer for historical browser pageviews (#40357)

Alan Guo Xiang Tan31ff0071f843
diff --git a/app/jobs/scheduled/aggregate_browser_pageview_daily_rollups.rb b/app/jobs/scheduled/aggregate_browser_pageview_daily_rollups.rb
deleted file mode 100644
index 664ac0af376..00000000000
--- a/app/jobs/scheduled/aggregate_browser_pageview_daily_rollups.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
- class AggregateBrowserPageviewDailyRollups < ::Jobs::Scheduled
- every 30.minutes
-
- LOCK_KEY = "aggregate_browser_pageview_daily_rollups"
-
- def execute(args)
- return if !SiteSetting.persist_browser_pageview_events
-
- start_date, end_date = aggregation_window
- return if start_date.nil?
-
- DistributedMutex.synchronize(LOCK_KEY, validity: 10.minutes) do
- BrowserPageviewCountryDailyRollup.aggregate(start_date: start_date, end_date: end_date)
- BrowserPageviewReferrerDailyRollup.aggregate(start_date: start_date, end_date: end_date)
- end
- end
-
- private
-
- def aggregation_window
- end_date = Time.zone.today
-
- if BrowserPageviewCountryDailyRollup.none? && BrowserPageviewReferrerDailyRollup.none?
- earliest_event_date = BrowserPageviewEvent.minimum(:created_at)&.to_date
- [earliest_event_date, end_date]
- else
- [1.day.ago.to_date, end_date]
- end
- end
- end
-end
diff --git a/app/jobs/scheduled/clean_up_browser_pageview_events.rb b/app/jobs/scheduled/clean_up_browser_pageview_events.rb
index 8e39d56b635..12ed20182f0 100644
--- a/app/jobs/scheduled/clean_up_browser_pageview_events.rb
+++ b/app/jobs/scheduled/clean_up_browser_pageview_events.rb
@@ -4,13 +4,11 @@ module Jobs
class CleanUpBrowserPageviewEvents < ::Jobs::Scheduled
every 1.day
- RETENTION_PERIOD = 3.months
-
def execute(args)
return if !SiteSetting.clean_up_browser_pageview_events
BrowserPageviewEvent
- .where("created_at < ?", RETENTION_PERIOD.ago.beginning_of_day)
+ .where("created_at < ?", BrowserPageviewEvent.retention_cutoff)
.in_batches(of: 10_000) do |browser_pageview_events|
BrowserPageviewEventScore.where(event_id: browser_pageview_events.select(:id)).delete_all
browser_pageview_events.delete_all
diff --git a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb
new file mode 100644
index 00000000000..d4d1f3e7fec
--- /dev/null
+++ b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb
@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module Jobs
+ class MaintainBrowserPageviewRollups < ::Jobs::Scheduled
+ every 10.minutes
+
+ cluster_concurrency 1
+
+ def execute(_args)
+ return if !SiteSetting.persist_browser_pageview_events
+
+ aggregate
+ backfill
+ end
+
+ private
+
+ def aggregate
+ start_date, end_date = aggregation_window
+ return if start_date.nil?
+
+ BrowserPageviewCountryDailyRollup.aggregate(start_date: start_date, end_date: end_date)
+ BrowserPageviewReferrerDailyRollup.aggregate(start_date: start_date, end_date: end_date)
+ end
+
+ def aggregation_window
+ end_date = Time.zone.today
+
+ if BrowserPageviewCountryDailyRollup.none? && BrowserPageviewReferrerDailyRollup.none?
+ earliest_event_date = BrowserPageviewEvent.minimum(:created_at)&.to_date
+ [earliest_event_date, end_date]
+ else
+ [1.day.ago.to_date, end_date]
+ end
+ end
+
+ def backfill
+ rows = next_batch
+ return if rows.empty?
+
+ ids = rows.map(&:id)
+
+ store_normalized_referrers(rows)
+ BrowserPageviewReferrerDailyRollup.recompute(recomputable_dates(ids))
+ stamp_version(ids)
+ end
+
+ def next_batch
+ params = { version: BrowserPageviewReferrerInspector::VERSION, limit: batch_size }
+
+ retention_clause = ""
+ if SiteSetting.clean_up_browser_pageview_events
+ retention_clause = "AND created_at >= :retention_cutoff"
+ # CleanUpBrowserPageviewEvents computes its own cutoff, so around
+ # midnight the two cutoffs can differ by a day. The extra day ensures
+ # the backfill never rebuilds a day that cleanup may be deleting.
+ params[:retention_cutoff] = BrowserPageviewEvent.retention_cutoff + 1.day
+ end
+
+ DB.query(<<~SQL, params)
+ SELECT id, referrer
+ FROM browser_pageview_events
+ WHERE referrer IS NOT NULL
+ AND (
+ normalized_referrer_version IS NULL
+ OR normalized_referrer_version < :version
+ )
+ #{retention_clause}
+ LIMIT :limit
+ SQL
+ end
+
+ def store_normalized_referrers(rows)
+ ids = rows.map(&:id)
+ normalized = rows.map { |row| BrowserPageviewReferrerInspector.normalize(row.referrer) }
+
+ DB.exec(<<~SQL, ids: ids, normalized: normalized)
+ UPDATE browser_pageview_events AS e
+ SET normalized_referrer = data.normalized_referrer
+ FROM (
+ SELECT
+ unnest(ARRAY[:ids]::bigint[]) AS id,
+ unnest(ARRAY[:normalized]::text[]) AS normalized_referrer
+ ) AS data
+ WHERE e.id = data.id
+ SQL
+ end
+
+ def recomputable_dates(ids)
+ params = { ids: ids, version: BrowserPageviewReferrerInspector::VERSION }
+
+ retention_clause = ""
+ if SiteSetting.clean_up_browser_pageview_events
+ retention_clause = "AND e.created_at >= :retention_cutoff"
+ params[:retention_cutoff] = BrowserPageviewEvent.retention_cutoff + 1.day
+ end
+
+ DB.query_single(<<~SQL, params)
+ WITH batch_ids AS (
+ SELECT unnest(ARRAY[:ids]::bigint[]) AS id
+ ),
+ touched_dates AS (
+ SELECT DISTINCT created_at::date AS date
+ FROM browser_pageview_events
+ WHERE id IN (:ids)
+ )
+ SELECT touched_dates.date
+ FROM touched_dates
+ WHERE NOT EXISTS (
+ SELECT 1
+ FROM browser_pageview_events e
+ WHERE e.created_at >= touched_dates.date
+ AND e.created_at < touched_dates.date + 1
+ AND e.referrer IS NOT NULL
+ AND NOT EXISTS (
+ SELECT 1
+ FROM batch_ids
+ WHERE batch_ids.id = e.id
+ )
+ AND (
+ e.normalized_referrer_version IS NULL
+ OR e.normalized_referrer_version < :version
+ )
+ #{retention_clause}
+ )
+ SQL
+ end
+
+ def stamp_version(ids)
+ DB.exec(<<~SQL, version: BrowserPageviewReferrerInspector::VERSION, ids: ids)
+ UPDATE browser_pageview_events
+ SET normalized_referrer_version = :version
+ WHERE id IN (:ids)
+ SQL
+ end
+
+ def batch_size
+ SiteSetting.browser_pageview_referrer_backfill_batch_size
+ end
+ end
+end
diff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rb
index 59179ae72f3..6499a650bff 100644
--- a/app/models/browser_pageview_event.rb
+++ b/app/models/browser_pageview_event.rb
@@ -6,9 +6,14 @@ class BrowserPageviewEvent < ActiveRecord::Base
MAX_REFERRER_LENGTH = 2000
MAX_USER_AGENT_LENGTH = 1000
MAX_NORMALIZED_REFERRER_LENGTH = 2000
+ RETENTION_PERIOD = 3.months
has_one :browser_pageview_event_score, foreign_key: :event_id, dependent: :delete
+ def self.retention_cutoff
+ RETENTION_PERIOD.ago.beginning_of_day
+ end
+
before_save :truncate_fields
private
@@ -28,25 +33,27 @@ end
#
# Table name: browser_pageview_events
#
-# id :bigint not null, primary key
-# asn :integer
-# country_code :string(2)
-# ip_address :inet not null
-# normalized_referrer :string(2000)
-# referrer :string(2000)
-# score :integer
-# url :string(2000) not null
-# user_agent :string(1000) not null
-# created_at :datetime not null
-# session_id :string(32) not null
-# topic_id :integer
-# user_id :integer
+# id :bigint not null, primary key
+# asn :integer
+# country_code :string(2)
+# ip_address :inet not null
+# normalized_referrer :string(2000)
+# normalized_referrer_version :integer
+# referrer :string(2000)
+# score :integer
+# url :string(2000) not null
+# user_agent :string(1000) not null
+# created_at :datetime not null
+# session_id :string(32) not null
+# topic_id :integer
+# user_id :integer
#
# Indexes
#
# idx_bpe_created_at_country_code (created_at,country_code)
# idx_bpe_created_at_normalized_referrer (created_at,normalized_referrer)
# idx_bpe_ip_ua_created_at (ip_address,user_agent,created_at)
+# idx_bpe_normalized_referrer_version (normalized_referrer_version) WHERE (referrer IS NOT NULL)
# idx_bpe_session_created_at (session_id,created_at)
# index_browser_pageview_events_on_created_at (created_at) USING brin
# index_browser_pageview_events_on_topic_id (topic_id)
diff --git a/app/models/browser_pageview_referrer_daily_rollup.rb b/app/models/browser_pageview_referrer_daily_rollup.rb
index e210d23172d..0cd32da319c 100644
--- a/app/models/browser_pageview_referrer_daily_rollup.rb
+++ b/app/models/browser_pageview_referrer_daily_rollup.rb
@@ -18,6 +18,36 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::Base
logged_in_count = EXCLUDED.logged_in_count
SQL
end
+
+ def self.recompute(dates)
+ dates = Array(dates).map(&:to_date).uniq
+ return if dates.empty?
+
+ # The rollups are the permanent record, but their source events are pruned
+ # after a retention period (CleanUpBrowserPageviewEvents). Only rebuild
+ # dates that still have events so we never delete a rollup we can no longer
+ # reconstruct from events.
+ dates = DB.query_single(<<~SQL, dates: dates)
+ SELECT d.date
+ FROM unnest(ARRAY[:dates]::date[]) AS d(date)
+ WHERE EXISTS (
+ SELECT 1
+ FROM browser_pageview_events e
+ WHERE e.created_at >= d.date
+ AND e.created_at < d.date + 1
+ )
+ SQL
+ return if dates.empty?
+
+ transaction do
+ DB.exec(<<~SQL, dates: dates)
+ DELETE FROM browser_pageview_referrer_daily_rollups
+ WHERE date IN (:dates)
+ SQL
+
+ dates.each { |date| aggregate(start_date: date, end_date: date) }
+ end
+ end
end
# == Schema Information
diff --git a/config/site_settings.yml b/config/site_settings.yml
index 804fb6eb491..1c4f2c939d0 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -614,6 +614,12 @@ basic:
clean_up_browser_pageview_events:
default: false
hidden: true
+ browser_pageview_referrer_backfill_batch_size:
+ default: 5000
+ type: integer
+ min: 1
+ max: 20000
+ hidden: true
experimental_detect_crawler_pageviews:
default: false
hidden: true
diff --git a/db/migrate/20260528074719_add_normalized_referrer_version_to_browser_pageview_events.rb b/db/migrate/20260528074719_add_normalized_referrer_version_to_browser_pageview_events.rb
new file mode 100644
index 00000000000..4c4171c83eb
--- /dev/null
+++ b/db/migrate/20260528074719_add_normalized_referrer_version_to_browser_pageview_events.rb
@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+class AddNormalizedReferrerVersionToBrowserPageviewEvents < ActiveRecord::Migration[8.0]
+ def change
+ add_column :browser_pageview_events, :normalized_referrer_version, :integer, limit: 2
+ end
+end
diff --git a/db/migrate/20260528074731_add_normalized_referrer_version_index_to_browser_pageview_events.rb b/db/migrate/20260528074731_add_normalized_referrer_version_index_to_browser_pageview_events.rb
new file mode 100644
index 00000000000..bb3ad4d240c
--- /dev/null
+++ b/db/migrate/20260528074731_add_normalized_referrer_version_index_to_browser_pageview_events.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+class AddNormalizedReferrerVersionIndexToBrowserPageviewEvents < ActiveRecord::Migration[8.0]
+ disable_ddl_transaction!
+
+ INDEX_NAME = "idx_bpe_normalized_referrer_version"
+
+ def up
+ remove_index :browser_pageview_events,
+ name: INDEX_NAME,
+ algorithm: :concurrently,
+ if_exists: true
+ add_index :browser_pageview_events,
+ :normalized_referrer_version,
+ name: INDEX_NAME,
+ where: "referrer IS NOT NULL",
+ algorithm: :concurrently
+ end
+
+ def down
+ remove_index :browser_pageview_events,
+ name: INDEX_NAME,
+ algorithm: :concurrently,
+ if_exists: true
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index 6f3af8d1ad7..977f118e283 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1950,7 +1950,8 @@ CREATE TABLE public.browser_pageview_events (
created_at timestamp without time zone NOT NULL,
asn integer,
score integer,
- normalized_referrer character varying(2000)
+ normalized_referrer character varying(2000),
+ normalized_referrer_version smallint
);
@@ -16563,6 +16564,13 @@ CREATE INDEX idx_bpe_created_at_normalized_referrer ON public.browser_pageview_e
CREATE INDEX idx_bpe_ip_ua_created_at ON public.browser_pageview_events USING btree (ip_address, user_agent, created_at);
+--
+-- Name: idx_bpe_normalized_referrer_version; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_bpe_normalized_referrer_version ON public.browser_pageview_events USING btree (normalized_referrer_version) WHERE (referrer IS NOT NULL);
+
+
--
-- Name: idx_bpe_session_created_at; Type: INDEX; Schema: public; Owner: -
--
@@ -21855,6 +21863,8 @@ INSERT INTO "schema_migrations" (version) VALUES
('20260603115200'),
('20260602104726'),
('20260601043020'),
+('20260528074731'),
+('20260528074719'),
('20260525105009'),
('20260525105006'),
('20260522043337'),
diff --git a/lib/browser_pageview_referrer_inspector.rb b/lib/browser_pageview_referrer_inspector.rb
index 21c8c0fe598..26c49b87bad 100644
--- a/lib/browser_pageview_referrer_inspector.rb
+++ b/lib/browser_pageview_referrer_inspector.rb
@@ -6,6 +6,10 @@
# query params, converts the host to lowercase punycode, and truncates the
# result to 200 bytes.
class BrowserPageviewReferrerInspector
+ # Bump when the normalization logic changes significantly to trigger a
+ # re-backfill of rows stamped with an older version.
+ VERSION = 1
+
# TODO: consider vendoring DuckDuckGo's Tracker Radar tracking-parameter list
# (https://github.com/duckduckgo/tracker-radar) for broader, maintained
# coverage instead of this hand-curated subset.
diff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rb
index e3fb40e90df..299dee4e099 100644
--- a/lib/middleware/request_tracker.rb
+++ b/lib/middleware/request_tracker.rb
@@ -735,6 +735,7 @@ class Middleware::RequestTracker
asn: payload[:asn],
referrer: payload[:referrer],
normalized_referrer: BrowserPageviewReferrerInspector.normalize(payload[:referrer]),
+ normalized_referrer_version: BrowserPageviewReferrerInspector::VERSION,
user_agent: payload[:user_agent],
session_id: payload[:session_id],
user_id: payload[:user_id],
diff --git a/spec/fabricators/browser_pageview_event_fabricator.rb b/spec/fabricators/browser_pageview_event_fabricator.rb
index 77eb7995b70..0e10e47bd62 100644
--- a/spec/fabricators/browser_pageview_event_fabricator.rb
+++ b/spec/fabricators/browser_pageview_event_fabricator.rb
@@ -6,3 +6,9 @@ Fabricator(:browser_pageview_event) do
user_agent "test"
session_id { SecureRandom.hex(16) }
end
+
+Fabricator(:browser_pageview_event_with_unnormalized_referrer, from: :browser_pageview_event) do
+ referrer "https://www.example.com/"
+ normalized_referrer nil
+ normalized_referrer_version nil
+end
diff --git a/spec/jobs/aggregate_browser_pageview_daily_rollups_spec.rb b/spec/jobs/aggregate_browser_pageview_daily_rollups_spec.rb
deleted file mode 100644
index 481cb6b6a8c..00000000000
--- a/spec/jobs/aggregate_browser_pageview_daily_rollups_spec.rb
+++ /dev/null
@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-RSpec.describe Jobs::AggregateBrowserPageviewDailyRollups do
- before { Discourse.stubs(:current_hostname).returns("forum.example.com") }
-
- it "does nothing when persist_browser_pageview_events is disabled" do
- SiteSetting.persist_browser_pageview_events = false
- Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")
-
- expect { described_class.new.execute({}) }.not_to change {
- BrowserPageviewCountryDailyRollup.count + BrowserPageviewReferrerDailyRollup.count
- }
- end
-
- it "aggregates both country and referrer rollups from recent pageview events" do
- SiteSetting.persist_browser_pageview_events = true
- Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")
-
- described_class.new.execute({})
-
- expect(BrowserPageviewCountryDailyRollup.where(country_code: "US").sum(:count)).to eq(1)
- expect(
- BrowserPageviewReferrerDailyRollup.where(normalized_referrer: "google.com").sum(:count),
- ).to eq(1)
- end
-
- it "backfills from the earliest event date on the first run when rollups are empty" do
- SiteSetting.persist_browser_pageview_events = true
- Fabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)
- Fabricate(:browser_pageview_event, country_code: "GB", created_at: 5.days.ago)
-
- described_class.new.execute({})
-
- expect(BrowserPageviewCountryDailyRollup.pluck(:country_code)).to contain_exactly("US", "GB")
- end
-
- it "only aggregates yesterday and today once historical rollups are populated" do
- SiteSetting.persist_browser_pageview_events = true
- Fabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)
- described_class.new.execute({}) # first run backfills everything
-
- Fabricate(:browser_pageview_event, country_code: "GB", created_at: 60.days.ago) # late old event
- Fabricate(:browser_pageview_event, country_code: "FR") # today
- described_class.new.execute({}) # second run only does yesterday + today
-
- expect(BrowserPageviewCountryDailyRollup.pluck(:country_code)).to contain_exactly("US", "FR")
- end
-end
diff --git a/spec/jobs/maintain_browser_pageview_rollups_spec.rb b/spec/jobs/maintain_browser_pageview_rollups_spec.rb
new file mode 100644
index 00000000000..93fb1598cbd
--- /dev/null
+++ b/spec/jobs/maintain_browser_pageview_rollups_spec.rb
@@ -0,0 +1,287 @@
+# frozen_string_literal: true
+
+RSpec.describe Jobs::MaintainBrowserPageviewRollups do
+ subject(:job) { described_class.new }
+
+ before { SiteSetting.persist_browser_pageview_events = true }
+
+ describe "#execute" do
+ it "does nothing when persist_browser_pageview_events is disabled" do
+ SiteSetting.persist_browser_pageview_events = false
+ event =
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: "https://reddit.com/",
+ country_code: "US",
+ )
+
+ job.execute({})
+
+ expect(BrowserPageviewCountryDailyRollup.count).to eq(0)
+ expect(BrowserPageviewReferrerDailyRollup.count).to eq(0)
+ expect(event.reload.normalized_referrer_version).to be_nil
+ end
+
+ context "when aggregating rollups" do
+ it "aggregates both country and referrer rollups from recent pageview events" do
+ Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")
+
+ job.execute({})
+
+ expect(BrowserPageviewCountryDailyRollup.where(country_code: "US").sum(:count)).to eq(1)
+ expect(
+ BrowserPageviewReferrerDailyRollup.where(normalized_referrer: "google.com").sum(:count),
+ ).to eq(1)
+ end
+
+ it "backfills from the earliest event date on the first run when rollups are empty" do
+ Fabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)
+ Fabricate(:browser_pageview_event, country_code: "GB", created_at: 5.days.ago)
+
+ job.execute({})
+
+ expect(BrowserPageviewCountryDailyRollup.pluck(:country_code)).to contain_exactly(
+ "US",
+ "GB",
+ )
+ end
+
+ it "only aggregates yesterday and today once historical rollups are populated" do
+ Fabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)
+ job.execute({}) # first run backfills everything
+
+ Fabricate(:browser_pageview_event, country_code: "GB", created_at: 60.days.ago) # late old event
+ Fabricate(:browser_pageview_event, country_code: "FR") # today
+ job.execute({}) # second run only does yesterday + today
+
+ expect(BrowserPageviewCountryDailyRollup.pluck(:country_code)).to contain_exactly(
+ "US",
+ "FR",
+ )
+ end
+ end
+
+ context "when backfilling referrers" do
+ it "normalizes historical rows using the inspector and stamps the current version" do
+ raw = "https://www.reddit.com/r/discourse/"
+ event = Fabricate(:browser_pageview_event_with_unnormalized_referrer, referrer: raw)
+
+ job.execute({})
+
+ expect(event.reload.normalized_referrer).to eq(
+ BrowserPageviewReferrerInspector.normalize(raw),
+ )
+ expect(event.normalized_referrer_version).to eq(BrowserPageviewReferrerInspector::VERSION)
+ end
+
+ it "leaves rows without a referrer untouched and does not let them block completion" do
+ direct_visit = Fabricate(:browser_pageview_event_with_unnormalized_referrer, referrer: nil)
+ referred =
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: "https://reddit.com/",
+ )
+
+ job.execute({})
+
+ expect(direct_visit.reload.normalized_referrer_version).to be_nil
+ expect(referred.reload.normalized_referrer_version).to eq(
+ BrowserPageviewReferrerInspector::VERSION,
+ )
+ expect { job.execute({}) }.not_to change { referred.reload.normalized_referrer_version }
+ end
+
+ it "processes an empty-string referrer once, normalizing it to NULL and stamping it" do
+ event = Fabricate(:browser_pageview_event_with_unnormalized_referrer, referrer: "")
+
+ job.execute({})
+
+ event.reload
+ expect(event.normalized_referrer).to be_nil
+ expect(event.normalized_referrer_version).to eq(BrowserPageviewReferrerInspector::VERSION)
+ end
+
+ it "is a no-op once every referrer row is current" do
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: "https://reddit.com/",
+ )
+ job.execute({})
+
+ expect { job.execute({}) }.not_to change {
+ BrowserPageviewReferrerDailyRollup.pluck(:id, :count)
+ }
+ end
+
+ it "only processes up to the configured batch size per run" do
+ SiteSetting.browser_pageview_referrer_backfill_batch_size = 1
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: "https://reddit.com/",
+ )
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: "https://news.ycombinator.com/",
+ )
+
+ job.execute({})
+
+ expect(BrowserPageviewEvent.where(normalized_referrer_version: nil).count).to eq(1)
+ end
+
+ it "waits to repair a date until every stale referrer row for that date is processed" do
+ SiteSetting.browser_pageview_referrer_backfill_batch_size = 1
+ date = 3.days.ago.to_date
+ 2.times do
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: "https://www.google.com/",
+ created_at: date,
+ )
+ end
+
+ job.execute({})
+
+ rollups =
+ BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count)
+ expect(rollups).to eq([[nil, 2]])
+
+ job.execute({})
+
+ rollups =
+ BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count)
+ expect(rollups).to eq([["google.com", 2]])
+ end
+
+ it "skips rows within a day of the retention cutoff when cleanup is enabled" do
+ SiteSetting.clean_up_browser_pageview_events = true
+ prunable =
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: "https://www.reddit.com/",
+ created_at: (BrowserPageviewEvent::RETENTION_PERIOD + 1.day).ago,
+ )
+ near_cutoff =
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: "https://www.reddit.com/",
+ created_at: BrowserPageviewEvent::RETENTION_PERIOD.ago,
+ )
+ recent =
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: "https://www.reddit.com/",
+ )
+
+ job.execute({})
+
+ prunable.reload
+ expect(prunable.normalized_referrer).to be_nil
+ expect(prunable.normalized_referrer_version).to be_nil
+ expect(near_cutoff.reload.normalized_referrer_version).to be_nil
+ expect(recent.reload.normalized_referrer_version).to eq(
+ BrowserPageviewReferrerInspector::VERSION,
+ )
+ end
+
+ it "backfills rows older than the retention period when cleanup is disabled" do
+ SiteSetting.clean_up_browser_pageview_events = false
+ event =
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: "https://www.reddit.com/",
+ created_at: (BrowserPageviewEvent::RETENTION_PERIOD + 1.day).ago,
+ )
+
+ job.execute({})
+
+ expect(event.reload.normalized_referrer_version).to eq(
+ BrowserPageviewReferrerInspector::VERSION,
+ )
+ end
+
+ it "repairs the affected rollups to match a fresh aggregation, including the NULL bucket" do
+ date = 3.days.ago.to_date
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: "https://www.google.com/",
+ created_at: date,
+ )
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: "https://www.google.com/",
+ created_at: date,
+ )
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: nil,
+ created_at: date,
+ )
+
+ job.execute({})
+
+ rollups =
+ BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count)
+ expect(rollups).to contain_exactly(["google.com", 2], [nil, 1])
+ end
+
+ it "re-selects the row and repairs the rollup when a crash happens before the version is stamped" do
+ date = 3.days.ago.to_date
+ event =
+ Fabricate(
+ :browser_pageview_event_with_unnormalized_referrer,
+ referrer: "https://www.google.com/",
+ created_at: date,
+ )
+
+ BrowserPageviewReferrerDailyRollup.stubs(:recompute).raises("rollup boom")
+ expect { job.execute({}) }.to raise_error("rollup boom")
+
+ expect(event.reload.normalized_referrer).to eq("google.com")
+ expect(event.normalized_referrer_version).to be_nil
+ expect(
+ BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count),
+ ).to eq([[nil, 1]])
+
+ BrowserPageviewReferrerDailyRollup.unstub(:recompute)
+ job.execute({})
+
+ expect(event.reload.normalized_referrer_version).to eq(
+ BrowserPageviewReferrerInspector::VERSION,
+ )
+ expect(
+ BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count),
+ ).to eq([["google.com", 1]])
+ end
+
+ it "re-normalizes rows stamped with an older version and repairs their rollups" do
+ date = 3.days.ago.to_date
+ raw = "https://google.com/?utm_source=newsletter"
+ event =
+ Fabricate(
+ :browser_pageview_event,
+ referrer: raw,
+ normalized_referrer: "google.com/?utm_source=newsletter",
+ normalized_referrer_version: BrowserPageviewReferrerInspector::VERSION,
+ created_at: date,
+ )
+
+ stub_const(
+ BrowserPageviewReferrerInspector,
+ "VERSION",
+ BrowserPageviewReferrerInspector::VERSION + 1,
+ ) do
+ job.execute({})
+
+ event.reload
+ expect(event.normalized_referrer).to eq(BrowserPageviewReferrerInspector.normalize(raw))
+ expect(event.normalized_referrer_version).to eq(BrowserPageviewReferrerInspector::VERSION)
+ end
+
+ expect(BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer)).to eq(
+ ["google.com"],
+ )
+ end
+ end
+ end
+end
diff --git a/spec/lib/middleware/request_tracker_spec.rb b/spec/lib/middleware/request_tracker_spec.rb
index 282403aedbb..805b90fb9b0 100644
--- a/spec/lib/middleware/request_tracker_spec.rb
+++ b/spec/lib/middleware/request_tracker_spec.rb
@@ -978,6 +978,7 @@ RSpec.describe Middleware::RequestTracker do
event = BrowserPageviewEvent.last
expect(event.referrer).to eq("https://www.example.com/path?utm_source=x")
expect(event.normalized_referrer).to eq("example.com/path")
+ expect(event.normalized_referrer_version).to eq(BrowserPageviewReferrerInspector::VERSION)
end
it "stores nil normalized_referrer when the referrer is blank" do
diff --git a/spec/models/browser_pageview_referrer_daily_rollup_spec.rb b/spec/models/browser_pageview_referrer_daily_rollup_spec.rb
index c1d6ae6aee1..c340cc65da2 100644
--- a/spec/models/browser_pageview_referrer_daily_rollup_spec.rb
+++ b/spec/models/browser_pageview_referrer_daily_rollup_spec.rb
@@ -82,4 +82,104 @@ RSpec.describe BrowserPageviewReferrerDailyRollup do
expect(described_class.sum(:count)).to eq(1)
end
end
+
+ describe ".recompute" do
+ it "produces a rollup row only for referrers that have events on the date" do
+ date = 2.days.ago.to_date
+ described_class.create!(date:, normalized_referrer: nil, count: 2, logged_in_count: 0)
+ 2.times do
+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com", created_at: date)
+ end
+
+ described_class.recompute([date])
+
+ expect(described_class.where(date:).pluck(:normalized_referrer, :count)).to contain_exactly(
+ ["google.com", 2],
+ )
+ end
+
+ it "produces a NULL rollup row for pageviews with no referrer" do
+ date = 2.days.ago.to_date
+ Fabricate(:browser_pageview_event, normalized_referrer: nil, created_at: date)
+ Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com", created_at: date)
+
+ described_class.recompute([date])
+
+ expect(described_class.where(date:).pluck(:normalized_referrer, :count)).to contain_exactly(
+ [nil, 1],
+ ["reddit.com", 1],
+ )
+ end
+
+ it "only recomputes the given dates" do
+ requested_date = 2.days.ago.to_date
+ untouched_date = 5.days.ago.to_date
+ described_class.create!(
+ date: untouched_date,
+ normalized_referrer: "stale.example.com",
+ count: 9,
+ logged_in_count: 4,
+ )
+ Fabricate(
+ :browser_pageview_event,
+ normalized_referrer: "google.com",
+ created_at: requested_date,
+ )
+ Fabricate(
+ :browser_pageview_event,
+ normalized_referrer: "bing.com",
+ created_at: untouched_date,
+ )
+
+ described_class.recompute([requested_date])
+
+ expect(described_class.where(date: requested_date).pluck(:normalized_referrer)).to eq(
+ ["google.com"],
+ )
+ expect(
+ described_class.where(date: untouched_date).pluck(
+ :normalized_referrer,
+ :count,
+ :logged_in_count,
+ ),
+ ).to eq([["stale.example.com", 9, 4]])
+ end
+
+ it "leaves an existing rollup untouched for a date whose events are gone" do
+ date = 2.days.ago.to_date
+ described_class.create!(
+ date:,
+ normalized_referrer: "google.com",
+ count: 5,
+ logged_in_count: 2,
+ )
+
+ described_class.recompute([date])
+
+ expect(
+ described_class.where(date:).pluck(:normalized_referrer, :count, :logged_in_count),
+ ).to eq([["google.com", 5, 2]])
+ end
+
+ it "rebuilds dates that have events while leaving dates without events untouched" do
+ date_live = 2.days.ago.to_date
+ date_gone = 5.days.ago.to_date
+ described_class.create!(
+ date: date_gone,
+ normalized_referrer: "stale.example.com",
+ count: 9,
+ logged_in_count: 0,
+ )
+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com", created_at: date_live)
+
+ described_class.recompute([date_gone, date_live])
+
+ expect(described_class.where(date: date_gone).pluck(:normalized_referrer, :count)).to eq(
+ [["stale.example.com", 9]],
+ )
+ expect(described_class.where(date: date_live).pluck(:normalized_referrer, :count)).to eq(
+ [["google.com", 1]],
+ )
+ end
+ end
end