A platform for community discussion. Free, open, simple.
PERF: Index browser pageview URL and referrer backfill lookups (#43651)
Keegan George638e2d56f07f
diff --git a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb b/app/jobs/scheduled/maintain_browser_pageview_rollups.rbindex ab616a6ad94..2992e2112bf 100644--- a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb+++ b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb@@ -139,6 +139,7 @@ module JobsOR normalized_referrer_version < :version)#{retention_clause}+ ORDER BY created_at DESC, id DESCLIMIT :limitSQLend@@ -340,7 +341,7 @@ module Jobsnormalized_url_version IS NULLOR normalized_url_version < :version)- ORDER BY id+ ORDER BY created_at DESC, id DESCLIMIT :limitSQLretention_cutoff: BrowserPageviewEvent.retention_cutoff,diff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rbindex a70c4338da1..887bc0c0023 100644--- a/app/models/browser_pageview_event.rb+++ b/app/models/browser_pageview_event.rb@@ -321,9 +321,9 @@ end# idx_bpe_created_at_country_code (created_at,country_code)# idx_bpe_created_at_normalized_referrer (created_at,normalized_referrer)# idx_bpe_ip_ua_created_at (ip_address,user_agent,created_at)-# idx_bpe_normalized_referrer_version (normalized_referrer_version) WHERE (referrer IS NOT NULL)-# idx_bpe_normalized_url_version (normalized_url_version)+# idx_bpe_referrer_backfill (created_at,id) WHERE ((referrer IS NOT NULL) AND ((normalized_referrer_version IS NULL) OR (normalized_referrer_version < 1)))# idx_bpe_session_created_at (session_id,created_at)+# idx_bpe_url_backfill (created_at,id) WHERE ((normalized_url_version IS NULL) OR (normalized_url_version < 1))# index_browser_pageview_events_on_created_at (created_at) USING brin# index_browser_pageview_events_on_topic_id (topic_id)# index_browser_pageview_events_on_user_id (user_id)diff --git a/db/migrate/20260914172757_add_backfill_indexes_to_browser_pageview_events.rb b/db/migrate/20260914172757_add_backfill_indexes_to_browser_pageview_events.rbnew file mode 100644index 00000000000..40dab24c88d--- /dev/null+++ b/db/migrate/20260914172757_add_backfill_indexes_to_browser_pageview_events.rb@@ -0,0 +1,50 @@+# frozen_string_literal: true++class AddBackfillIndexesToBrowserPageviewEvents < ActiveRecord::Migration[8.0]+ disable_ddl_transaction!++ URL_INDEX_NAME = "idx_bpe_url_backfill"+ REFERRER_INDEX_NAME = "idx_bpe_referrer_backfill"++ def up+ remove_index :browser_pageview_events,+ name: URL_INDEX_NAME,+ algorithm: :concurrently,+ if_exists: true+ add_index :browser_pageview_events,+ %i[created_at id],+ order: {+ created_at: :desc,+ id: :desc,+ },+ where: "normalized_url_version IS NULL OR normalized_url_version < 1",+ name: URL_INDEX_NAME,+ algorithm: :concurrently++ remove_index :browser_pageview_events,+ name: REFERRER_INDEX_NAME,+ algorithm: :concurrently,+ if_exists: true+ add_index :browser_pageview_events,+ %i[created_at id],+ order: {+ created_at: :desc,+ id: :desc,+ },+ where:+ "referrer IS NOT NULL AND (normalized_referrer_version IS NULL OR normalized_referrer_version < 1)",+ name: REFERRER_INDEX_NAME,+ algorithm: :concurrently+ end++ def down+ remove_index :browser_pageview_events,+ name: URL_INDEX_NAME,+ algorithm: :concurrently,+ if_exists: true+ remove_index :browser_pageview_events,+ name: REFERRER_INDEX_NAME,+ algorithm: :concurrently,+ if_exists: true+ end+enddiff --git a/db/post_migrate/20260914172801_drop_unused_browser_pageview_version_indexes.rb b/db/post_migrate/20260914172801_drop_unused_browser_pageview_version_indexes.rbnew file mode 100644index 00000000000..10fe35d73a6--- /dev/null+++ b/db/post_migrate/20260914172801_drop_unused_browser_pageview_version_indexes.rb@@ -0,0 +1,28 @@+# frozen_string_literal: true++class DropUnusedBrowserPageviewVersionIndexes < ActiveRecord::Migration[8.0]+ disable_ddl_transaction!++ def up+ remove_index :browser_pageview_events,+ name: "idx_bpe_normalized_url_version",+ algorithm: :concurrently,+ if_exists: true+ remove_index :browser_pageview_events,+ name: "idx_bpe_normalized_referrer_version",+ algorithm: :concurrently,+ if_exists: true+ end++ def down+ add_index :browser_pageview_events,+ :normalized_url_version,+ name: "idx_bpe_normalized_url_version",+ algorithm: :concurrently+ add_index :browser_pageview_events,+ :normalized_referrer_version,+ name: "idx_bpe_normalized_referrer_version",+ where: "referrer IS NOT NULL",+ algorithm: :concurrently+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex 02e8c9a362b..4e8806edb91 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -19010,24 +19010,24 @@ CREATE INDEX idx_bpe_ip_ua_created_at ON public.browser_pageview_events USING bt----- Name: idx_bpe_normalized_referrer_version; Type: INDEX; Schema: public; Owner: -+-- Name: idx_bpe_referrer_backfill; Type: INDEX; Schema: public; Owner: ----CREATE INDEX idx_bpe_normalized_referrer_version ON public.browser_pageview_events USING btree (normalized_referrer_version) WHERE (referrer IS NOT NULL);+CREATE INDEX idx_bpe_referrer_backfill ON public.browser_pageview_events USING btree (created_at DESC, id DESC) WHERE ((referrer IS NOT NULL) AND ((normalized_referrer_version IS NULL) OR (normalized_referrer_version < 1)));----- Name: idx_bpe_normalized_url_version; Type: INDEX; Schema: public; Owner: -+-- Name: idx_bpe_session_created_at; Type: INDEX; Schema: public; Owner: ----CREATE INDEX idx_bpe_normalized_url_version ON public.browser_pageview_events USING btree (normalized_url_version);+CREATE INDEX idx_bpe_session_created_at ON public.browser_pageview_events USING btree (session_id, created_at);----- Name: idx_bpe_session_created_at; Type: INDEX; Schema: public; Owner: -+-- Name: idx_bpe_url_backfill; Type: INDEX; Schema: public; Owner: ----CREATE INDEX idx_bpe_session_created_at ON public.browser_pageview_events USING btree (session_id, created_at);+CREATE INDEX idx_bpe_url_backfill ON public.browser_pageview_events USING btree (created_at DESC, id DESC) WHERE ((normalized_url_version IS NULL) OR (normalized_url_version < 1));--@@ -25072,6 +25072,8 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groupsSET search_path TO "$user", public;INSERT INTO "schema_migrations" (version) VALUES+('20260914172801'),+('20260914172757'),('20260910033302'),('20260908160656'),('20260908153158'),diff --git a/lib/browser_pageview_event_url_normalizer.rb b/lib/browser_pageview_event_url_normalizer.rbindex 744cf50272c..73995f57386 100644--- a/lib/browser_pageview_event_url_normalizer.rb+++ b/lib/browser_pageview_event_url_normalizer.rb@@ -8,11 +8,13 @@# paths for the Site Traffic Explorer.class BrowserPageviewEventUrlNormalizer# Bump when referrer normalization changes significantly to re-backfill rows- # stamped with an older version.+ # stamped with an older version. Also recreate idx_bpe_referrer_backfill,+ # whose predicate hardcodes this value.REFERRER_VERSION = 1# Bump when site-path normalization changes significantly to re-backfill rows- # stamped with an older version.+ # stamped with an older version. Also recreate idx_bpe_url_backfill, whose+ # predicate hardcodes this value.SITE_PATH_VERSION = 1# TODO: consider vendoring DuckDuckGo's Tracker Radar tracking-parameter listdiff --git a/spec/jobs/maintain_browser_pageview_rollups_spec.rb b/spec/jobs/maintain_browser_pageview_rollups_spec.rbindex 7406ed27897..4067b246008 100644--- a/spec/jobs/maintain_browser_pageview_rollups_spec.rb+++ b/spec/jobs/maintain_browser_pageview_rollups_spec.rb@@ -643,7 +643,7 @@ RSpec.describe Jobs::MaintainBrowserPageviewRollups doexpect(BrowserPageviewEvent.where(normalized_url_version: nil).pluck(:id),- ).to contain_exactly(second.id)+ ).to contain_exactly(first.id)job.execute({})