A platform for community discussion. Free, open, simple.
FIX: Backfill normalized referrer for historical browser pageviews (#40357)
Alan Guo Xiang Tan31ff0071f843
diff --git a/app/jobs/scheduled/aggregate_browser_pageview_daily_rollups.rb b/app/jobs/scheduled/aggregate_browser_pageview_daily_rollups.rbdeleted file mode 100644index 664ac0af376..00000000000--- a/app/jobs/scheduled/aggregate_browser_pageview_daily_rollups.rb+++ /dev/null@@ -1,34 +0,0 @@-# frozen_string_literal: true--module Jobs- class AggregateBrowserPageviewDailyRollups < ::Jobs::Scheduled- every 30.minutes-- LOCK_KEY = "aggregate_browser_pageview_daily_rollups"-- def execute(args)- return if !SiteSetting.persist_browser_pageview_events-- start_date, end_date = aggregation_window- return if start_date.nil?-- DistributedMutex.synchronize(LOCK_KEY, validity: 10.minutes) do- BrowserPageviewCountryDailyRollup.aggregate(start_date: start_date, end_date: end_date)- BrowserPageviewReferrerDailyRollup.aggregate(start_date: start_date, end_date: end_date)- end- end-- private-- def aggregation_window- end_date = Time.zone.today-- if BrowserPageviewCountryDailyRollup.none? && BrowserPageviewReferrerDailyRollup.none?- earliest_event_date = BrowserPageviewEvent.minimum(:created_at)&.to_date- [earliest_event_date, end_date]- else- [1.day.ago.to_date, end_date]- end- end- end-enddiff --git a/app/jobs/scheduled/clean_up_browser_pageview_events.rb b/app/jobs/scheduled/clean_up_browser_pageview_events.rbindex 8e39d56b635..12ed20182f0 100644--- a/app/jobs/scheduled/clean_up_browser_pageview_events.rb+++ b/app/jobs/scheduled/clean_up_browser_pageview_events.rb@@ -4,13 +4,11 @@ module Jobsclass CleanUpBrowserPageviewEvents < ::Jobs::Scheduledevery 1.day- RETENTION_PERIOD = 3.months-def execute(args)return if !SiteSetting.clean_up_browser_pageview_eventsBrowserPageviewEvent- .where("created_at < ?", RETENTION_PERIOD.ago.beginning_of_day)+ .where("created_at < ?", BrowserPageviewEvent.retention_cutoff).in_batches(of: 10_000) do |browser_pageview_events|BrowserPageviewEventScore.where(event_id: browser_pageview_events.select(:id)).delete_allbrowser_pageview_events.delete_alldiff --git a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb b/app/jobs/scheduled/maintain_browser_pageview_rollups.rbnew file mode 100644index 00000000000..d4d1f3e7fec--- /dev/null+++ b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb@@ -0,0 +1,141 @@+# frozen_string_literal: true++module Jobs+ class MaintainBrowserPageviewRollups < ::Jobs::Scheduled+ every 10.minutes++ cluster_concurrency 1++ def execute(_args)+ return if !SiteSetting.persist_browser_pageview_events++ aggregate+ backfill+ end++ private++ def aggregate+ start_date, end_date = aggregation_window+ return if start_date.nil?++ BrowserPageviewCountryDailyRollup.aggregate(start_date: start_date, end_date: end_date)+ BrowserPageviewReferrerDailyRollup.aggregate(start_date: start_date, end_date: end_date)+ end++ def aggregation_window+ end_date = Time.zone.today++ if BrowserPageviewCountryDailyRollup.none? && BrowserPageviewReferrerDailyRollup.none?+ earliest_event_date = BrowserPageviewEvent.minimum(:created_at)&.to_date+ [earliest_event_date, end_date]+ else+ [1.day.ago.to_date, end_date]+ end+ end++ def backfill+ rows = next_batch+ return if rows.empty?++ ids = rows.map(&:id)++ store_normalized_referrers(rows)+ BrowserPageviewReferrerDailyRollup.recompute(recomputable_dates(ids))+ stamp_version(ids)+ end++ def next_batch+ params = { version: BrowserPageviewReferrerInspector::VERSION, limit: batch_size }++ retention_clause = ""+ if SiteSetting.clean_up_browser_pageview_events+ retention_clause = "AND created_at >= :retention_cutoff"+ # CleanUpBrowserPageviewEvents computes its own cutoff, so around+ # midnight the two cutoffs can differ by a day. The extra day ensures+ # the backfill never rebuilds a day that cleanup may be deleting.+ params[:retention_cutoff] = BrowserPageviewEvent.retention_cutoff + 1.day+ end++ DB.query(<<~SQL, params)+ SELECT id, referrer+ FROM browser_pageview_events+ WHERE referrer IS NOT NULL+ AND (+ normalized_referrer_version IS NULL+ OR normalized_referrer_version < :version+ )+ #{retention_clause}+ LIMIT :limit+ SQL+ end++ def store_normalized_referrers(rows)+ ids = rows.map(&:id)+ normalized = rows.map { |row| BrowserPageviewReferrerInspector.normalize(row.referrer) }++ DB.exec(<<~SQL, ids: ids, normalized: normalized)+ UPDATE browser_pageview_events AS e+ SET normalized_referrer = data.normalized_referrer+ FROM (+ SELECT+ unnest(ARRAY[:ids]::bigint[]) AS id,+ unnest(ARRAY[:normalized]::text[]) AS normalized_referrer+ ) AS data+ WHERE e.id = data.id+ SQL+ end++ def recomputable_dates(ids)+ params = { ids: ids, version: BrowserPageviewReferrerInspector::VERSION }++ retention_clause = ""+ if SiteSetting.clean_up_browser_pageview_events+ retention_clause = "AND e.created_at >= :retention_cutoff"+ params[:retention_cutoff] = BrowserPageviewEvent.retention_cutoff + 1.day+ end++ DB.query_single(<<~SQL, params)+ WITH batch_ids AS (+ SELECT unnest(ARRAY[:ids]::bigint[]) AS id+ ),+ touched_dates AS (+ SELECT DISTINCT created_at::date AS date+ FROM browser_pageview_events+ WHERE id IN (:ids)+ )+ SELECT touched_dates.date+ FROM touched_dates+ WHERE NOT EXISTS (+ SELECT 1+ FROM browser_pageview_events e+ WHERE e.created_at >= touched_dates.date+ AND e.created_at < touched_dates.date + 1+ AND e.referrer IS NOT NULL+ AND NOT EXISTS (+ SELECT 1+ FROM batch_ids+ WHERE batch_ids.id = e.id+ )+ AND (+ e.normalized_referrer_version IS NULL+ OR e.normalized_referrer_version < :version+ )+ #{retention_clause}+ )+ SQL+ end++ def stamp_version(ids)+ DB.exec(<<~SQL, version: BrowserPageviewReferrerInspector::VERSION, ids: ids)+ UPDATE browser_pageview_events+ SET normalized_referrer_version = :version+ WHERE id IN (:ids)+ SQL+ end++ def batch_size+ SiteSetting.browser_pageview_referrer_backfill_batch_size+ end+ end+enddiff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rbindex 59179ae72f3..6499a650bff 100644--- a/app/models/browser_pageview_event.rb+++ b/app/models/browser_pageview_event.rb@@ -6,9 +6,14 @@ class BrowserPageviewEvent < ActiveRecord::BaseMAX_REFERRER_LENGTH = 2000MAX_USER_AGENT_LENGTH = 1000MAX_NORMALIZED_REFERRER_LENGTH = 2000+ RETENTION_PERIOD = 3.monthshas_one :browser_pageview_event_score, foreign_key: :event_id, dependent: :delete+ def self.retention_cutoff+ RETENTION_PERIOD.ago.beginning_of_day+ end+before_save :truncate_fieldsprivate@@ -28,25 +33,27 @@ end## Table name: browser_pageview_events#-# id :bigint not null, primary key-# asn :integer-# country_code :string(2)-# ip_address :inet not null-# normalized_referrer :string(2000)-# referrer :string(2000)-# score :integer-# url :string(2000) not null-# user_agent :string(1000) not null-# created_at :datetime not null-# session_id :string(32) not null-# topic_id :integer-# user_id :integer+# id :bigint not null, primary key+# asn :integer+# country_code :string(2)+# ip_address :inet not null+# normalized_referrer :string(2000)+# normalized_referrer_version :integer+# referrer :string(2000)+# score :integer+# url :string(2000) not null+# user_agent :string(1000) not null+# created_at :datetime not null+# session_id :string(32) not null+# topic_id :integer+# user_id :integer## Indexes## idx_bpe_created_at_country_code (created_at,country_code)# idx_bpe_created_at_normalized_referrer (created_at,normalized_referrer)# idx_bpe_ip_ua_created_at (ip_address,user_agent,created_at)+# idx_bpe_normalized_referrer_version (normalized_referrer_version) WHERE (referrer IS NOT NULL)# idx_bpe_session_created_at (session_id,created_at)# index_browser_pageview_events_on_created_at (created_at) USING brin# index_browser_pageview_events_on_topic_id (topic_id)diff --git a/app/models/browser_pageview_referrer_daily_rollup.rb b/app/models/browser_pageview_referrer_daily_rollup.rbindex e210d23172d..0cd32da319c 100644--- a/app/models/browser_pageview_referrer_daily_rollup.rb+++ b/app/models/browser_pageview_referrer_daily_rollup.rb@@ -18,6 +18,36 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::Baselogged_in_count = EXCLUDED.logged_in_countSQLend++ def self.recompute(dates)+ dates = Array(dates).map(&:to_date).uniq+ return if dates.empty?++ # The rollups are the permanent record, but their source events are pruned+ # after a retention period (CleanUpBrowserPageviewEvents). Only rebuild+ # dates that still have events so we never delete a rollup we can no longer+ # reconstruct from events.+ dates = DB.query_single(<<~SQL, dates: dates)+ SELECT d.date+ FROM unnest(ARRAY[:dates]::date[]) AS d(date)+ WHERE EXISTS (+ SELECT 1+ FROM browser_pageview_events e+ WHERE e.created_at >= d.date+ AND e.created_at < d.date + 1+ )+ SQL+ return if dates.empty?++ transaction do+ DB.exec(<<~SQL, dates: dates)+ DELETE FROM browser_pageview_referrer_daily_rollups+ WHERE date IN (:dates)+ SQL++ dates.each { |date| aggregate(start_date: date, end_date: date) }+ end+ endend# == Schema Informationdiff --git a/config/site_settings.yml b/config/site_settings.ymlindex 804fb6eb491..1c4f2c939d0 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -614,6 +614,12 @@ basic:clean_up_browser_pageview_events:default: falsehidden: true+ browser_pageview_referrer_backfill_batch_size:+ default: 5000+ type: integer+ min: 1+ max: 20000+ hidden: trueexperimental_detect_crawler_pageviews:default: falsehidden: truediff --git a/db/migrate/20260528074719_add_normalized_referrer_version_to_browser_pageview_events.rb b/db/migrate/20260528074719_add_normalized_referrer_version_to_browser_pageview_events.rbnew file mode 100644index 00000000000..4c4171c83eb--- /dev/null+++ b/db/migrate/20260528074719_add_normalized_referrer_version_to_browser_pageview_events.rb@@ -0,0 +1,6 @@+# frozen_string_literal: true+class AddNormalizedReferrerVersionToBrowserPageviewEvents < ActiveRecord::Migration[8.0]+ def change+ add_column :browser_pageview_events, :normalized_referrer_version, :integer, limit: 2+ end+enddiff --git a/db/migrate/20260528074731_add_normalized_referrer_version_index_to_browser_pageview_events.rb b/db/migrate/20260528074731_add_normalized_referrer_version_index_to_browser_pageview_events.rbnew file mode 100644index 00000000000..bb3ad4d240c--- /dev/null+++ b/db/migrate/20260528074731_add_normalized_referrer_version_index_to_browser_pageview_events.rb@@ -0,0 +1,25 @@+# frozen_string_literal: true+class AddNormalizedReferrerVersionIndexToBrowserPageviewEvents < ActiveRecord::Migration[8.0]+ disable_ddl_transaction!++ INDEX_NAME = "idx_bpe_normalized_referrer_version"++ def up+ remove_index :browser_pageview_events,+ name: INDEX_NAME,+ algorithm: :concurrently,+ if_exists: true+ add_index :browser_pageview_events,+ :normalized_referrer_version,+ name: INDEX_NAME,+ where: "referrer IS NOT NULL",+ algorithm: :concurrently+ end++ def down+ remove_index :browser_pageview_events,+ name: INDEX_NAME,+ algorithm: :concurrently,+ if_exists: true+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex 6f3af8d1ad7..977f118e283 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -1950,7 +1950,8 @@ CREATE TABLE public.browser_pageview_events (created_at timestamp without time zone NOT NULL,asn integer,score integer,- normalized_referrer character varying(2000)+ normalized_referrer character varying(2000),+ normalized_referrer_version smallint);@@ -16563,6 +16564,13 @@ CREATE INDEX idx_bpe_created_at_normalized_referrer ON public.browser_pageview_eCREATE INDEX idx_bpe_ip_ua_created_at ON public.browser_pageview_events USING btree (ip_address, user_agent, created_at);+--+-- Name: idx_bpe_normalized_referrer_version; Type: INDEX; Schema: public; Owner: -+--++CREATE INDEX idx_bpe_normalized_referrer_version ON public.browser_pageview_events USING btree (normalized_referrer_version) WHERE (referrer IS NOT NULL);++---- Name: idx_bpe_session_created_at; Type: INDEX; Schema: public; Owner: ---@@ -21855,6 +21863,8 @@ INSERT INTO "schema_migrations" (version) VALUES('20260603115200'),('20260602104726'),('20260601043020'),+('20260528074731'),+('20260528074719'),('20260525105009'),('20260525105006'),('20260522043337'),diff --git a/lib/browser_pageview_referrer_inspector.rb b/lib/browser_pageview_referrer_inspector.rbindex 21c8c0fe598..26c49b87bad 100644--- a/lib/browser_pageview_referrer_inspector.rb+++ b/lib/browser_pageview_referrer_inspector.rb@@ -6,6 +6,10 @@# query params, converts the host to lowercase punycode, and truncates the# result to 200 bytes.class BrowserPageviewReferrerInspector+ # Bump when the normalization logic changes significantly to trigger a+ # re-backfill of rows stamped with an older version.+ VERSION = 1+# TODO: consider vendoring DuckDuckGo's Tracker Radar tracking-parameter list# (https://github.com/duckduckgo/tracker-radar) for broader, maintained# coverage instead of this hand-curated subset.diff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rbindex e3fb40e90df..299dee4e099 100644--- a/lib/middleware/request_tracker.rb+++ b/lib/middleware/request_tracker.rb@@ -735,6 +735,7 @@ class Middleware::RequestTrackerasn: payload[:asn],referrer: payload[:referrer],normalized_referrer: BrowserPageviewReferrerInspector.normalize(payload[:referrer]),+ normalized_referrer_version: BrowserPageviewReferrerInspector::VERSION,user_agent: payload[:user_agent],session_id: payload[:session_id],user_id: payload[:user_id],diff --git a/spec/fabricators/browser_pageview_event_fabricator.rb b/spec/fabricators/browser_pageview_event_fabricator.rbindex 77eb7995b70..0e10e47bd62 100644--- a/spec/fabricators/browser_pageview_event_fabricator.rb+++ b/spec/fabricators/browser_pageview_event_fabricator.rb@@ -6,3 +6,9 @@ Fabricator(:browser_pageview_event) douser_agent "test"session_id { SecureRandom.hex(16) }end++Fabricator(:browser_pageview_event_with_unnormalized_referrer, from: :browser_pageview_event) do+ referrer "https://www.example.com/"+ normalized_referrer nil+ normalized_referrer_version nil+enddiff --git a/spec/jobs/aggregate_browser_pageview_daily_rollups_spec.rb b/spec/jobs/aggregate_browser_pageview_daily_rollups_spec.rbdeleted file mode 100644index 481cb6b6a8c..00000000000--- a/spec/jobs/aggregate_browser_pageview_daily_rollups_spec.rb+++ /dev/null@@ -1,48 +0,0 @@-# frozen_string_literal: true--RSpec.describe Jobs::AggregateBrowserPageviewDailyRollups do- before { Discourse.stubs(:current_hostname).returns("forum.example.com") }-- it "does nothing when persist_browser_pageview_events is disabled" do- SiteSetting.persist_browser_pageview_events = false- Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")-- expect { described_class.new.execute({}) }.not_to change {- BrowserPageviewCountryDailyRollup.count + BrowserPageviewReferrerDailyRollup.count- }- end-- it "aggregates both country and referrer rollups from recent pageview events" do- SiteSetting.persist_browser_pageview_events = true- Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")-- described_class.new.execute({})-- expect(BrowserPageviewCountryDailyRollup.where(country_code: "US").sum(:count)).to eq(1)- expect(- BrowserPageviewReferrerDailyRollup.where(normalized_referrer: "google.com").sum(:count),- ).to eq(1)- end-- it "backfills from the earliest event date on the first run when rollups are empty" do- SiteSetting.persist_browser_pageview_events = true- Fabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)- Fabricate(:browser_pageview_event, country_code: "GB", created_at: 5.days.ago)-- described_class.new.execute({})-- expect(BrowserPageviewCountryDailyRollup.pluck(:country_code)).to contain_exactly("US", "GB")- end-- it "only aggregates yesterday and today once historical rollups are populated" do- SiteSetting.persist_browser_pageview_events = true- Fabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)- described_class.new.execute({}) # first run backfills everything-- Fabricate(:browser_pageview_event, country_code: "GB", created_at: 60.days.ago) # late old event- Fabricate(:browser_pageview_event, country_code: "FR") # today- described_class.new.execute({}) # second run only does yesterday + today-- expect(BrowserPageviewCountryDailyRollup.pluck(:country_code)).to contain_exactly("US", "FR")- end-enddiff --git a/spec/jobs/maintain_browser_pageview_rollups_spec.rb b/spec/jobs/maintain_browser_pageview_rollups_spec.rbnew file mode 100644index 00000000000..93fb1598cbd--- /dev/null+++ b/spec/jobs/maintain_browser_pageview_rollups_spec.rb@@ -0,0 +1,287 @@+# frozen_string_literal: true++RSpec.describe Jobs::MaintainBrowserPageviewRollups do+ subject(:job) { described_class.new }++ before { SiteSetting.persist_browser_pageview_events = true }++ describe "#execute" do+ it "does nothing when persist_browser_pageview_events is disabled" do+ SiteSetting.persist_browser_pageview_events = false+ event =+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: "https://reddit.com/",+ country_code: "US",+ )++ job.execute({})++ expect(BrowserPageviewCountryDailyRollup.count).to eq(0)+ expect(BrowserPageviewReferrerDailyRollup.count).to eq(0)+ expect(event.reload.normalized_referrer_version).to be_nil+ end++ context "when aggregating rollups" do+ it "aggregates both country and referrer rollups from recent pageview events" do+ Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")++ job.execute({})++ expect(BrowserPageviewCountryDailyRollup.where(country_code: "US").sum(:count)).to eq(1)+ expect(+ BrowserPageviewReferrerDailyRollup.where(normalized_referrer: "google.com").sum(:count),+ ).to eq(1)+ end++ it "backfills from the earliest event date on the first run when rollups are empty" do+ Fabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)+ Fabricate(:browser_pageview_event, country_code: "GB", created_at: 5.days.ago)++ job.execute({})++ expect(BrowserPageviewCountryDailyRollup.pluck(:country_code)).to contain_exactly(+ "US",+ "GB",+ )+ end++ it "only aggregates yesterday and today once historical rollups are populated" do+ Fabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)+ job.execute({}) # first run backfills everything++ Fabricate(:browser_pageview_event, country_code: "GB", created_at: 60.days.ago) # late old event+ Fabricate(:browser_pageview_event, country_code: "FR") # today+ job.execute({}) # second run only does yesterday + today++ expect(BrowserPageviewCountryDailyRollup.pluck(:country_code)).to contain_exactly(+ "US",+ "FR",+ )+ end+ end++ context "when backfilling referrers" do+ it "normalizes historical rows using the inspector and stamps the current version" do+ raw = "https://www.reddit.com/r/discourse/"+ event = Fabricate(:browser_pageview_event_with_unnormalized_referrer, referrer: raw)++ job.execute({})++ expect(event.reload.normalized_referrer).to eq(+ BrowserPageviewReferrerInspector.normalize(raw),+ )+ expect(event.normalized_referrer_version).to eq(BrowserPageviewReferrerInspector::VERSION)+ end++ it "leaves rows without a referrer untouched and does not let them block completion" do+ direct_visit = Fabricate(:browser_pageview_event_with_unnormalized_referrer, referrer: nil)+ referred =+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: "https://reddit.com/",+ )++ job.execute({})++ expect(direct_visit.reload.normalized_referrer_version).to be_nil+ expect(referred.reload.normalized_referrer_version).to eq(+ BrowserPageviewReferrerInspector::VERSION,+ )+ expect { job.execute({}) }.not_to change { referred.reload.normalized_referrer_version }+ end++ it "processes an empty-string referrer once, normalizing it to NULL and stamping it" do+ event = Fabricate(:browser_pageview_event_with_unnormalized_referrer, referrer: "")++ job.execute({})++ event.reload+ expect(event.normalized_referrer).to be_nil+ expect(event.normalized_referrer_version).to eq(BrowserPageviewReferrerInspector::VERSION)+ end++ it "is a no-op once every referrer row is current" do+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: "https://reddit.com/",+ )+ job.execute({})++ expect { job.execute({}) }.not_to change {+ BrowserPageviewReferrerDailyRollup.pluck(:id, :count)+ }+ end++ it "only processes up to the configured batch size per run" do+ SiteSetting.browser_pageview_referrer_backfill_batch_size = 1+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: "https://reddit.com/",+ )+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: "https://news.ycombinator.com/",+ )++ job.execute({})++ expect(BrowserPageviewEvent.where(normalized_referrer_version: nil).count).to eq(1)+ end++ it "waits to repair a date until every stale referrer row for that date is processed" do+ SiteSetting.browser_pageview_referrer_backfill_batch_size = 1+ date = 3.days.ago.to_date+ 2.times do+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: "https://www.google.com/",+ created_at: date,+ )+ end++ job.execute({})++ rollups =+ BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count)+ expect(rollups).to eq([[nil, 2]])++ job.execute({})++ rollups =+ BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count)+ expect(rollups).to eq([["google.com", 2]])+ end++ it "skips rows within a day of the retention cutoff when cleanup is enabled" do+ SiteSetting.clean_up_browser_pageview_events = true+ prunable =+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: "https://www.reddit.com/",+ created_at: (BrowserPageviewEvent::RETENTION_PERIOD + 1.day).ago,+ )+ near_cutoff =+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: "https://www.reddit.com/",+ created_at: BrowserPageviewEvent::RETENTION_PERIOD.ago,+ )+ recent =+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: "https://www.reddit.com/",+ )++ job.execute({})++ prunable.reload+ expect(prunable.normalized_referrer).to be_nil+ expect(prunable.normalized_referrer_version).to be_nil+ expect(near_cutoff.reload.normalized_referrer_version).to be_nil+ expect(recent.reload.normalized_referrer_version).to eq(+ BrowserPageviewReferrerInspector::VERSION,+ )+ end++ it "backfills rows older than the retention period when cleanup is disabled" do+ SiteSetting.clean_up_browser_pageview_events = false+ event =+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: "https://www.reddit.com/",+ created_at: (BrowserPageviewEvent::RETENTION_PERIOD + 1.day).ago,+ )++ job.execute({})++ expect(event.reload.normalized_referrer_version).to eq(+ BrowserPageviewReferrerInspector::VERSION,+ )+ end++ it "repairs the affected rollups to match a fresh aggregation, including the NULL bucket" do+ date = 3.days.ago.to_date+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: "https://www.google.com/",+ created_at: date,+ )+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: "https://www.google.com/",+ created_at: date,+ )+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: nil,+ created_at: date,+ )++ job.execute({})++ rollups =+ BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count)+ expect(rollups).to contain_exactly(["google.com", 2], [nil, 1])+ end++ it "re-selects the row and repairs the rollup when a crash happens before the version is stamped" do+ date = 3.days.ago.to_date+ event =+ Fabricate(+ :browser_pageview_event_with_unnormalized_referrer,+ referrer: "https://www.google.com/",+ created_at: date,+ )++ BrowserPageviewReferrerDailyRollup.stubs(:recompute).raises("rollup boom")+ expect { job.execute({}) }.to raise_error("rollup boom")++ expect(event.reload.normalized_referrer).to eq("google.com")+ expect(event.normalized_referrer_version).to be_nil+ expect(+ BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count),+ ).to eq([[nil, 1]])++ BrowserPageviewReferrerDailyRollup.unstub(:recompute)+ job.execute({})++ expect(event.reload.normalized_referrer_version).to eq(+ BrowserPageviewReferrerInspector::VERSION,+ )+ expect(+ BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count),+ ).to eq([["google.com", 1]])+ end++ it "re-normalizes rows stamped with an older version and repairs their rollups" do+ date = 3.days.ago.to_date+ raw = "https://google.com/?utm_source=newsletter"+ event =+ Fabricate(+ :browser_pageview_event,+ referrer: raw,+ normalized_referrer: "google.com/?utm_source=newsletter",+ normalized_referrer_version: BrowserPageviewReferrerInspector::VERSION,+ created_at: date,+ )++ stub_const(+ BrowserPageviewReferrerInspector,+ "VERSION",+ BrowserPageviewReferrerInspector::VERSION + 1,+ ) do+ job.execute({})++ event.reload+ expect(event.normalized_referrer).to eq(BrowserPageviewReferrerInspector.normalize(raw))+ expect(event.normalized_referrer_version).to eq(BrowserPageviewReferrerInspector::VERSION)+ end++ expect(BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer)).to eq(+ ["google.com"],+ )+ end+ end+ end+enddiff --git a/spec/lib/middleware/request_tracker_spec.rb b/spec/lib/middleware/request_tracker_spec.rbindex 282403aedbb..805b90fb9b0 100644--- a/spec/lib/middleware/request_tracker_spec.rb+++ b/spec/lib/middleware/request_tracker_spec.rb@@ -978,6 +978,7 @@ RSpec.describe Middleware::RequestTracker doevent = BrowserPageviewEvent.lastexpect(event.referrer).to eq("https://www.example.com/path?utm_source=x")expect(event.normalized_referrer).to eq("example.com/path")+ expect(event.normalized_referrer_version).to eq(BrowserPageviewReferrerInspector::VERSION)endit "stores nil normalized_referrer when the referrer is blank" dodiff --git a/spec/models/browser_pageview_referrer_daily_rollup_spec.rb b/spec/models/browser_pageview_referrer_daily_rollup_spec.rbindex c1d6ae6aee1..c340cc65da2 100644--- a/spec/models/browser_pageview_referrer_daily_rollup_spec.rb+++ b/spec/models/browser_pageview_referrer_daily_rollup_spec.rb@@ -82,4 +82,104 @@ RSpec.describe BrowserPageviewReferrerDailyRollup doexpect(described_class.sum(:count)).to eq(1)endend++ describe ".recompute" do+ it "produces a rollup row only for referrers that have events on the date" do+ date = 2.days.ago.to_date+ described_class.create!(date:, normalized_referrer: nil, count: 2, logged_in_count: 0)+ 2.times do+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com", created_at: date)+ end++ described_class.recompute([date])++ expect(described_class.where(date:).pluck(:normalized_referrer, :count)).to contain_exactly(+ ["google.com", 2],+ )+ end++ it "produces a NULL rollup row for pageviews with no referrer" do+ date = 2.days.ago.to_date+ Fabricate(:browser_pageview_event, normalized_referrer: nil, created_at: date)+ Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com", created_at: date)++ described_class.recompute([date])++ expect(described_class.where(date:).pluck(:normalized_referrer, :count)).to contain_exactly(+ [nil, 1],+ ["reddit.com", 1],+ )+ end++ it "only recomputes the given dates" do+ requested_date = 2.days.ago.to_date+ untouched_date = 5.days.ago.to_date+ described_class.create!(+ date: untouched_date,+ normalized_referrer: "stale.example.com",+ count: 9,+ logged_in_count: 4,+ )+ Fabricate(+ :browser_pageview_event,+ normalized_referrer: "google.com",+ created_at: requested_date,+ )+ Fabricate(+ :browser_pageview_event,+ normalized_referrer: "bing.com",+ created_at: untouched_date,+ )++ described_class.recompute([requested_date])++ expect(described_class.where(date: requested_date).pluck(:normalized_referrer)).to eq(+ ["google.com"],+ )+ expect(+ described_class.where(date: untouched_date).pluck(+ :normalized_referrer,+ :count,+ :logged_in_count,+ ),+ ).to eq([["stale.example.com", 9, 4]])+ end++ it "leaves an existing rollup untouched for a date whose events are gone" do+ date = 2.days.ago.to_date+ described_class.create!(+ date:,+ normalized_referrer: "google.com",+ count: 5,+ logged_in_count: 2,+ )++ described_class.recompute([date])++ expect(+ described_class.where(date:).pluck(:normalized_referrer, :count, :logged_in_count),+ ).to eq([["google.com", 5, 2]])+ end++ it "rebuilds dates that have events while leaving dates without events untouched" do+ date_live = 2.days.ago.to_date+ date_gone = 5.days.ago.to_date+ described_class.create!(+ date: date_gone,+ normalized_referrer: "stale.example.com",+ count: 9,+ logged_in_count: 0,+ )+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com", created_at: date_live)++ described_class.recompute([date_gone, date_live])++ expect(described_class.where(date: date_gone).pluck(:normalized_referrer, :count)).to eq(+ [["stale.example.com", 9]],+ )+ expect(described_class.where(date: date_live).pluck(:normalized_referrer, :count)).to eq(+ [["google.com", 1]],+ )+ end+ endend