Skip to content

A platform for community discussion. Free, open, simple.

PERF: Index browser pageview URL and referrer backfill lookups (#43651)

Keegan George638e2d56f07f
diff --git a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb
index ab616a6ad94..2992e2112bf 100644
--- a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb
+++ b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb
@@ -139,6 +139,7 @@ module Jobs
OR normalized_referrer_version < :version
)
#{retention_clause}
+ ORDER BY created_at DESC, id DESC
LIMIT :limit
SQL
end
@@ -340,7 +341,7 @@ module Jobs
normalized_url_version IS NULL
OR normalized_url_version < :version
)
- ORDER BY id
+ ORDER BY created_at DESC, id DESC
LIMIT :limit
SQL
retention_cutoff: BrowserPageviewEvent.retention_cutoff,
diff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rb
index a70c4338da1..887bc0c0023 100644
--- a/app/models/browser_pageview_event.rb
+++ b/app/models/browser_pageview_event.rb
@@ -321,9 +321,9 @@ end
# idx_bpe_created_at_country_code (created_at,country_code)
# idx_bpe_created_at_normalized_referrer (created_at,normalized_referrer)
# idx_bpe_ip_ua_created_at (ip_address,user_agent,created_at)
-# idx_bpe_normalized_referrer_version (normalized_referrer_version) WHERE (referrer IS NOT NULL)
-# idx_bpe_normalized_url_version (normalized_url_version)
+# idx_bpe_referrer_backfill (created_at,id) WHERE ((referrer IS NOT NULL) AND ((normalized_referrer_version IS NULL) OR (normalized_referrer_version < 1)))
# idx_bpe_session_created_at (session_id,created_at)
+# idx_bpe_url_backfill (created_at,id) WHERE ((normalized_url_version IS NULL) OR (normalized_url_version < 1))
# index_browser_pageview_events_on_created_at (created_at) USING brin
# index_browser_pageview_events_on_topic_id (topic_id)
# index_browser_pageview_events_on_user_id (user_id)
diff --git a/db/migrate/20260914172757_add_backfill_indexes_to_browser_pageview_events.rb b/db/migrate/20260914172757_add_backfill_indexes_to_browser_pageview_events.rb
new file mode 100644
index 00000000000..40dab24c88d
--- /dev/null
+++ b/db/migrate/20260914172757_add_backfill_indexes_to_browser_pageview_events.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+class AddBackfillIndexesToBrowserPageviewEvents < ActiveRecord::Migration[8.0]
+ disable_ddl_transaction!
+
+ URL_INDEX_NAME = "idx_bpe_url_backfill"
+ REFERRER_INDEX_NAME = "idx_bpe_referrer_backfill"
+
+ def up
+ remove_index :browser_pageview_events,
+ name: URL_INDEX_NAME,
+ algorithm: :concurrently,
+ if_exists: true
+ add_index :browser_pageview_events,
+ %i[created_at id],
+ order: {
+ created_at: :desc,
+ id: :desc,
+ },
+ where: "normalized_url_version IS NULL OR normalized_url_version < 1",
+ name: URL_INDEX_NAME,
+ algorithm: :concurrently
+
+ remove_index :browser_pageview_events,
+ name: REFERRER_INDEX_NAME,
+ algorithm: :concurrently,
+ if_exists: true
+ add_index :browser_pageview_events,
+ %i[created_at id],
+ order: {
+ created_at: :desc,
+ id: :desc,
+ },
+ where:
+ "referrer IS NOT NULL AND (normalized_referrer_version IS NULL OR normalized_referrer_version < 1)",
+ name: REFERRER_INDEX_NAME,
+ algorithm: :concurrently
+ end
+
+ def down
+ remove_index :browser_pageview_events,
+ name: URL_INDEX_NAME,
+ algorithm: :concurrently,
+ if_exists: true
+ remove_index :browser_pageview_events,
+ name: REFERRER_INDEX_NAME,
+ algorithm: :concurrently,
+ if_exists: true
+ end
+end
diff --git a/db/post_migrate/20260914172801_drop_unused_browser_pageview_version_indexes.rb b/db/post_migrate/20260914172801_drop_unused_browser_pageview_version_indexes.rb
new file mode 100644
index 00000000000..10fe35d73a6
--- /dev/null
+++ b/db/post_migrate/20260914172801_drop_unused_browser_pageview_version_indexes.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class DropUnusedBrowserPageviewVersionIndexes < ActiveRecord::Migration[8.0]
+ disable_ddl_transaction!
+
+ def up
+ remove_index :browser_pageview_events,
+ name: "idx_bpe_normalized_url_version",
+ algorithm: :concurrently,
+ if_exists: true
+ remove_index :browser_pageview_events,
+ name: "idx_bpe_normalized_referrer_version",
+ algorithm: :concurrently,
+ if_exists: true
+ end
+
+ def down
+ add_index :browser_pageview_events,
+ :normalized_url_version,
+ name: "idx_bpe_normalized_url_version",
+ algorithm: :concurrently
+ add_index :browser_pageview_events,
+ :normalized_referrer_version,
+ name: "idx_bpe_normalized_referrer_version",
+ where: "referrer IS NOT NULL",
+ algorithm: :concurrently
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index 02e8c9a362b..4e8806edb91 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -19010,24 +19010,24 @@ CREATE INDEX idx_bpe_ip_ua_created_at ON public.browser_pageview_events USING bt
--
--- Name: idx_bpe_normalized_referrer_version; Type: INDEX; Schema: public; Owner: -
+-- Name: idx_bpe_referrer_backfill; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX idx_bpe_normalized_referrer_version ON public.browser_pageview_events USING btree (normalized_referrer_version) WHERE (referrer IS NOT NULL);
+CREATE INDEX idx_bpe_referrer_backfill ON public.browser_pageview_events USING btree (created_at DESC, id DESC) WHERE ((referrer IS NOT NULL) AND ((normalized_referrer_version IS NULL) OR (normalized_referrer_version < 1)));
--
--- Name: idx_bpe_normalized_url_version; Type: INDEX; Schema: public; Owner: -
+-- Name: idx_bpe_session_created_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX idx_bpe_normalized_url_version ON public.browser_pageview_events USING btree (normalized_url_version);
+CREATE INDEX idx_bpe_session_created_at ON public.browser_pageview_events USING btree (session_id, created_at);
--
--- Name: idx_bpe_session_created_at; Type: INDEX; Schema: public; Owner: -
+-- Name: idx_bpe_url_backfill; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX idx_bpe_session_created_at ON public.browser_pageview_events USING btree (session_id, created_at);
+CREATE INDEX idx_bpe_url_backfill ON public.browser_pageview_events USING btree (created_at DESC, id DESC) WHERE ((normalized_url_version IS NULL) OR (normalized_url_version < 1));
--
@@ -25072,6 +25072,8 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groups
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
+('20260914172801'),
+('20260914172757'),
('20260910033302'),
('20260908160656'),
('20260908153158'),
diff --git a/lib/browser_pageview_event_url_normalizer.rb b/lib/browser_pageview_event_url_normalizer.rb
index 744cf50272c..73995f57386 100644
--- a/lib/browser_pageview_event_url_normalizer.rb
+++ b/lib/browser_pageview_event_url_normalizer.rb
@@ -8,11 +8,13 @@
# paths for the Site Traffic Explorer.
class BrowserPageviewEventUrlNormalizer
# Bump when referrer normalization changes significantly to re-backfill rows
- # stamped with an older version.
+ # stamped with an older version. Also recreate idx_bpe_referrer_backfill,
+ # whose predicate hardcodes this value.
REFERRER_VERSION = 1
# Bump when site-path normalization changes significantly to re-backfill rows
- # stamped with an older version.
+ # stamped with an older version. Also recreate idx_bpe_url_backfill, whose
+ # predicate hardcodes this value.
SITE_PATH_VERSION = 1
# TODO: consider vendoring DuckDuckGo's Tracker Radar tracking-parameter list
diff --git a/spec/jobs/maintain_browser_pageview_rollups_spec.rb b/spec/jobs/maintain_browser_pageview_rollups_spec.rb
index 7406ed27897..4067b246008 100644
--- a/spec/jobs/maintain_browser_pageview_rollups_spec.rb
+++ b/spec/jobs/maintain_browser_pageview_rollups_spec.rb
@@ -643,7 +643,7 @@ RSpec.describe Jobs::MaintainBrowserPageviewRollups do
expect(
BrowserPageviewEvent.where(normalized_url_version: nil).pluck(:id),
- ).to contain_exactly(second.id)
+ ).to contain_exactly(first.id)
job.execute({})