A platform for community discussion. Free, open, simple.
PERF: Optimize site traffic explorer SQL query (#42547)
Alan Guo Xiang Tan2b6ae5fa0b0c
diff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rbindex d62153ac9c9..be71fdf1e42 100644--- a/app/models/browser_pageview_event.rb+++ b/app/models/browser_pageview_event.rb@@ -216,10 +216,12 @@ class BrowserPageviewEvent < ActiveRecord::BaseRETENTION_PERIOD.ago.beginning_of_dayend- def self.rollup_source_condition(table: nil)+ def self.rollup_source_condition(table: nil, start_date: nil, end_date: nil)prefix = table ? "#{table}." : ""cutover_date = beacon_cutover_datereturn "#{prefix}source = #{SOURCE_PIGGYBACK}" if cutover_date.nil?+ return "#{prefix}source = #{SOURCE_PIGGYBACK}" if end_date && end_date.to_date <= cutover_date+ return "#{prefix}source = #{SOURCE_BEACON}" if start_date && start_date.to_date >= cutover_datesanitize_sql_array([@@ -273,6 +275,7 @@ end## Indexes#+# idx_bpe_beacon_created_at_id (created_at,id) WHERE (source = 2)# idx_bpe_created_at_country_code (created_at,country_code)# idx_bpe_created_at_normalized_referrer (created_at,normalized_referrer)# idx_bpe_ip_ua_created_at (ip_address,user_agent,created_at)diff --git a/app/services/admin_dashboard_site_traffic_explorer.rb b/app/services/admin_dashboard_site_traffic_explorer.rbindex ac3d0827f19..15d9120c737 100644--- a/app/services/admin_dashboard_site_traffic_explorer.rb+++ b/app/services/admin_dashboard_site_traffic_explorer.rb@@ -118,10 +118,14 @@ class AdminDashboardSiteTrafficExplorerenddef execute_query(start_date:, end_date:, filters:)+ parameters = query_params(start_date:, end_date:, filters:)+ActiveRecord::Base.transaction(requires_new: true) do- DB.exec("SET TRANSACTION READ ONLY")DB.exec("SET LOCAL statement_timeout = #{STATEMENT_TIMEOUT_MS}")- DB.query_hash(query, query_params(start_date:, end_date:, filters:)).first+ DB.query_hash(+ query(start_date: parameters[:start_date], end_date: parameters[:end_date]),+ parameters,+ ).firstendend@@ -132,10 +136,9 @@ class AdminDashboardSiteTrafficExplorer{start_date: effective_start_date,- retention_cutoff: retention_cutoff,end_date: end_date + 1.day,cap: cap,- cap_plus_one: cap + 1,+ site_host: BrowserPageviewEventUrlNormalizer.normalize_host(Discourse.current_hostname),top_url: filters[:top_url],entry_url: filters[:entry_url],referrer: filters[:referrer],@@ -155,78 +158,48 @@ class AdminDashboardSiteTrafficExplorer}end- def query+ def query(start_date:, end_date:)+ source_condition =+ BrowserPageviewEvent.rollup_source_condition(table: "bpe", start_date:, end_date:)+<<~SQL- WITH eligible AS MATERIALIZED (+ WITH population AS MATERIALIZED (SELECT- candidates.id,- candidates.created_at,- candidates.session_id,- candidates.user_id,- candidates.normalized_url,- candidates.country_code,- candidates.asn,- candidates.ip_address,- candidates.user_agent,- candidates.score,- ROW_NUMBER() OVER (- ORDER BY candidates.created_at DESC, candidates.id DESC- ) AS cap_position- FROM (- SELECT- bpe.id,- bpe.created_at,- bpe.session_id,- bpe.user_id,- bpe.normalized_url,- bpe.country_code,- bpe.asn,- bpe.ip_address,- bpe.user_agent,- bpe.score- FROM browser_pageview_events bpe- WHERE bpe.created_at >= :start_date- AND bpe.created_at < :end_date- AND #{BrowserPageviewEvent.rollup_source_condition(table: "bpe")}- ORDER BY bpe.created_at DESC, bpe.id DESC- LIMIT :cap_plus_one- ) candidates+ bpe.id,+ bpe.created_at,+ bpe.session_id,+ bpe.user_id,+ bpe.normalized_url,+ bpe.normalized_referrer,+ bpe.country_code,+ bpe.asn,+ bpe.ip_address,+ bpe.user_agent,+ bpe.score+ FROM browser_pageview_events bpe+ WHERE bpe.created_at >= :start_date+ AND bpe.created_at < :end_date+ AND #{source_condition}+ ORDER BY bpe.created_at DESC, bpe.id DESC+ LIMIT :cap),- population AS NOT MATERIALIZED (+ population_stats AS MATERIALIZED (SELECT- id,- created_at,- session_id,- user_id,- normalized_url,- country_code,- asn,- ip_address,- user_agent,- score- FROM eligible- WHERE cap_position <= :cap- ),- population_sessions AS (- SELECT DISTINCT session_id+ COUNT(*) AS row_count,+ MIN(created_at) AS oldest_created_atFROM population),- session_entries AS (- SELECT entry.*- FROM population_sessions- CROSS JOIN LATERAL (- SELECT- bpe.id,- bpe.session_id,- bpe.normalized_url,- bpe.normalized_referrer- FROM browser_pageview_events bpe- WHERE bpe.session_id = population_sessions.session_id- AND bpe.created_at >= :retention_cutoff- AND #{BrowserPageviewEvent.rollup_source_condition(table: "bpe")}- ORDER BY bpe.created_at, bpe.id- LIMIT 1- ) entry+ population_boundary AS MATERIALIZED (+ SELECT+ population_stats.row_count,+ population_stats.oldest_created_at,+ MIN(population.id) AS oldest_id+ FROM population_stats+ LEFT JOIN population+ ON population.created_at = population_stats.oldest_created_at+ GROUP BY+ population_stats.row_count,+ population_stats.oldest_created_at),browser_values AS MATERIALIZED (SELECT@@ -244,15 +217,23 @@ class AdminDashboardSiteTrafficExplorer),classified AS (SELECT- population.id,population.created_at,population.session_id,population.user_id,population.normalized_url,+ population.normalized_referrer,population.country_code,population.asn,population.ip_address,browser_values.browser,+ (+ population.normalized_referrer IS NULL+ OR split_part(+ split_part(population.normalized_referrer, '/', 1),+ '?',+ 1+ ) <> :site_host+ ) AS acquisition_entry,(:crawler_detection_enabledAND COALESCE(population.score, 0) > :crawler_threshold@@ -263,17 +244,23 @@ class AdminDashboardSiteTrafficExplorer),dimensioned AS (SELECT- classified.*,+ classified.created_at,+ classified.session_id,+ classified.user_id,+ classified.normalized_url,+ classified.country_code,+ classified.asn,+ classified.ip_address,+ classified.browser,+ classified.likely_crawler,CASE- WHEN classified.id = session_entries.id THEN session_entries.normalized_url+ WHEN classified.acquisition_entry THEN classified.normalized_urlEND AS entry_url,CASE- WHEN session_entries.id IS NULL OR classified.id <> session_entries.id THEN NULL- ELSE COALESCE(session_entries.normalized_referrer, '')+ WHEN classified.acquisition_entry+ THEN COALESCE(classified.normalized_referrer, '')END AS referrerFROM classified- LEFT JOIN session_entries- ON session_entries.session_id = classified.session_id),filtered AS MATERIALIZED (SELECT *@@ -326,16 +313,6 @@ class AdminDashboardSiteTrafficExplorerLEFT JOIN browser_pageview_session_engagements engagementON engagement.session_id = sessions.session_id),- traffic_summary AS (- SELECT- COUNT(*) FILTER (- WHERE :traffic_type_filtered OR NOT likely_crawler- )::integer AS pageviews,- COUNT(*) FILTER (- WHERE NOT likely_crawler AND user_id IS NOT NULL- )::integer AS logged_in_pageviews- FROM filtered- ),series_rows AS (SELECTcreated_at::date AS date,@@ -351,18 +328,53 @@ class AdminDashboardSiteTrafficExplorerCOUNT(*) FILTER (WHERE likely_crawler)::integer AS likely_crawler_pageviewsFROM filteredGROUP BY created_at::date+ ),+ traffic_summary AS (+ SELECT+ COALESCE(SUM(pageviews), 0)::integer AS pageviews,+ COALESCE(SUM(logged_in_human_pageviews), 0)::integer AS logged_in_pageviews+ FROM series_rows+ ),+ bounded_dimension_rows AS MATERIALIZED (+ SELECT+ CASE+ WHEN GROUPING(country_code) = 0 THEN 'countries'+ WHEN GROUPING(browser) = 0 THEN 'browsers'+ END AS dimension,+ country_code,+ browser,+ COUNT(*)::integer AS pageviews,+ MIN(ip_address) AS representative_ip+ FROM filtered+ GROUP BY GROUPING SETS ((country_code), (browser)))SELECT- (SELECT EXISTS (SELECT 1 FROM eligible WHERE cap_position > :cap)) AS pageview_limited,+ (+ SELECT+ population_boundary.row_count = :cap+ AND EXISTS (+ SELECT 1+ FROM browser_pageview_events bpe+ WHERE bpe.created_at >= :start_date+ AND bpe.created_at < :end_date+ AND #{source_condition}+ AND (bpe.created_at, bpe.id) < (+ population_boundary.oldest_created_at,+ population_boundary.oldest_id+ )+ LIMIT 1+ )+ FROM population_boundary+ ) AS pageview_limited,jsonb_build_object('country', (- SELECT MIN(host(ip_address))+ SELECT host(MIN(ip_address))FROM populationWHERE :country IS NOT NULLAND country_code = :country),'network', (- SELECT MIN(host(ip_address))+ SELECT host(MIN(ip_address))FROM populationWHERE :network_asn IS NOT NULLAND asn = :network_asn@@ -476,11 +488,11 @@ class AdminDashboardSiteTrafficExplorerFROM (SELECTcountry_code AS value,- COUNT(*)::integer AS pageviews,- MIN(host(ip_address)) AS representative_ip- FROM filtered- WHERE country_code IS NOT NULL- GROUP BY country_code+ pageviews,+ host(representative_ip) AS representative_ip+ FROM bounded_dimension_rows+ WHERE dimension = 'countries'+ AND country_code IS NOT NULLORDER BY pageviews DESC, valueLIMIT :dimension_limit) rows@@ -501,7 +513,7 @@ class AdminDashboardSiteTrafficExplorerSELECT'AS' || asn AS value,COUNT(*)::integer AS pageviews,- MIN(host(ip_address)) AS representative_ip+ host(MIN(ip_address)) AS representative_ipFROM filteredWHERE asn IS NOT NULLGROUP BY asn@@ -518,9 +530,9 @@ class AdminDashboardSiteTrafficExplorer'[]'::jsonb)FROM (- SELECT browser AS value, COUNT(*)::integer AS pageviews- FROM filtered- GROUP BY browser+ SELECT browser AS value, pageviews+ FROM bounded_dimension_rows+ WHERE dimension = 'browsers'ORDER BY pageviews DESC, valueLIMIT :dimension_limit) rowsdiff --git a/db/migrate/20260812094609_add_beacon_traffic_explorer_index_to_browser_pageview_events.rb b/db/migrate/20260812094609_add_beacon_traffic_explorer_index_to_browser_pageview_events.rbnew file mode 100644index 00000000000..43a51eb258f--- /dev/null+++ b/db/migrate/20260812094609_add_beacon_traffic_explorer_index_to_browser_pageview_events.rb@@ -0,0 +1,30 @@+# frozen_string_literal: true+class AddBeaconTrafficExplorerIndexToBrowserPageviewEvents < ActiveRecord::Migration[8.0]+ disable_ddl_transaction!++ INDEX_NAME = "idx_bpe_beacon_created_at_id"+ private_constant :INDEX_NAME++ def up+ remove_index :browser_pageview_events,+ name: INDEX_NAME,+ algorithm: :concurrently,+ if_exists: true+ add_index :browser_pageview_events,+ %i[created_at id],+ name: INDEX_NAME,+ order: {+ created_at: :desc,+ id: :desc,+ },+ where: "source = 2",+ algorithm: :concurrently+ end++ def down+ remove_index :browser_pageview_events,+ name: INDEX_NAME,+ algorithm: :concurrently,+ if_exists: true+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex 6d1cecf1f6f..0e07dbaa594 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -17584,6 +17584,13 @@ CREATE UNIQUE INDEX idx_bpcd_rollups_date_country_unique ON public.browser_pagevCREATE UNIQUE INDEX idx_bpcrawler_rollups_date_logged_in_unique ON public.browser_pageview_crawler_daily_rollups USING btree (date, logged_in);+--+-- Name: idx_bpe_beacon_created_at_id; Type: INDEX; Schema: public; Owner: -+--++CREATE INDEX idx_bpe_beacon_created_at_id ON public.browser_pageview_events USING btree (created_at DESC, id DESC) WHERE (source = 2);++---- Name: idx_bpe_created_at_country_code; Type: INDEX; Schema: public; Owner: ---@@ -23151,6 +23158,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groupsSET search_path TO "$user", public;INSERT INTO "schema_migrations" (version) VALUES+('20260812094609'),('20260810154331'),('20260810012238'),('20260807182856'),diff --git a/spec/models/browser_pageview_event_spec.rb b/spec/models/browser_pageview_event_spec.rbindex 40e4542c6cf..f3eff9a23be 100644--- a/spec/models/browser_pageview_event_spec.rb+++ b/spec/models/browser_pageview_event_spec.rb@@ -99,6 +99,16 @@ RSpec.describe BrowserPageviewEvent dopiggyback = Fabricate(:browser_pageview_event, source: :piggyback)Fabricate(:browser_pageview_event, source: :beacon)+ bounded_condition =+ described_class.rollup_source_condition(+ start_date: Date.new(2026, 6, 1),+ end_date: Date.new(2026, 6, 20),+ )++ expect(described_class.rollup_source_condition).to eq(+ "source = #{described_class::SOURCE_PIGGYBACK}",+ )+ expect(bounded_condition).to eq("source = #{described_class::SOURCE_PIGGYBACK}")expect(described_class.where(described_class.rollup_source_condition)).to contain_exactly(piggyback,)@@ -131,6 +141,45 @@ RSpec.describe BrowserPageviewEvent doexpect(condition).to eq("e.source = #{described_class::SOURCE_PIGGYBACK}")end++ it "matches only piggyback events when the bounded range ends at the cutover" do+ described_class.stubs(:beacon_cutover_date).returns(Date.new(2026, 6, 10))++ condition =+ described_class.rollup_source_condition(+ start_date: Date.new(2026, 6, 1),+ end_date: Date.new(2026, 6, 10),+ )++ expect(condition).to eq("source = #{described_class::SOURCE_PIGGYBACK}")+ end++ it "matches only beacon events when the bounded range starts at the cutover" do+ described_class.stubs(:beacon_cutover_date).returns(Date.new(2026, 6, 10))++ condition =+ described_class.rollup_source_condition(+ start_date: Date.new(2026, 6, 10),+ end_date: Date.new(2026, 6, 20),+ )++ expect(condition).to eq("source = #{described_class::SOURCE_BEACON}")+ end++ it "keeps the mixed source condition when the bounded range crosses the cutover" do+ described_class.stubs(:beacon_cutover_date).returns(Date.new(2026, 6, 10))++ condition =+ described_class.rollup_source_condition(+ start_date: Date.new(2026, 6, 1),+ end_date: Date.new(2026, 6, 20),+ )++ expect(condition).to include(+ "created_at < '2026-06-10' AND source = #{described_class::SOURCE_PIGGYBACK}",+ "created_at >= '2026-06-10' AND source = #{described_class::SOURCE_BEACON}",+ )+ endendit "truncates string fields before saving" dodiff --git a/spec/requests/admin/dashboard_controller_spec.rb b/spec/requests/admin/dashboard_controller_spec.rbindex 7c45faad306..18fea810156 100644--- a/spec/requests/admin/dashboard_controller_spec.rb+++ b/spec/requests/admin/dashboard_controller_spec.rb@@ -1432,7 +1432,7 @@ RSpec.describe Admin::DashboardController douser_agent: "Mozilla/5.0 AppleWebKit/537.36 Chrome/124.0.0.0 Safari/537.36",user_id: admin.id,session_id: "admin-session",- normalized_referrer: "ignored.example/return?from=latest",+ normalized_referrer: "test.localhost/landing",normalized_referrer_version: BrowserPageviewEventUrlNormalizer::REFERRER_VERSION,source: BrowserPageviewEvent::SOURCE_BEACON,created_at: "2026-05-10 10:01:00",@@ -1584,6 +1584,38 @@ RSpec.describe Admin::DashboardController do},)end++ it "treats same-site referrers as internal navigation" do+ same_site_pageview =+ Fabricate(+ :browser_pageview_event,+ url: "https://test.localhost/same-site-full-load",+ normalized_referrer: "test.localhost/previous-page",+ session_id: "same-site-full-load",+ source: BrowserPageviewEvent::SOURCE_BEACON,+ created_at: "2026-05-11 11:00:00",+ )++ get "/admin/dashboard/site-traffic-explorer.json", params: request_params++ dimensions = response.parsed_body.fetch("dimensions")+ top_url =+ dimensions+ .fetch("top_urls")+ .find { |row| row["value"] == same_site_pageview.normalized_url }+ entry_url =+ dimensions+ .fetch("entry_urls")+ .find { |row| row["value"] == same_site_pageview.normalized_url }+ referrer =+ dimensions+ .fetch("referrers")+ .find { |row| row["value"] == same_site_pageview.normalized_referrer }++ expect([response.status, top_url&.fetch("pageviews"), entry_url, referrer]).to eq(+ [200, 1, nil, nil],+ )+ endendcontext "when the selected date range exceeds retention" do@@ -1720,7 +1752,7 @@ RSpec.describe Admin::DashboardController dobefore dosign_in(admin)- SiteSetting.stubs(:admin_site_traffic_event_cap).returns(2)+ SiteSetting.admin_site_traffic_event_cap = 2endit "returns no more than the configured pageview cap" do@@ -1815,7 +1847,7 @@ RSpec.describe Admin::DashboardController doit "returns both partial-data reasons" dosign_in(admin)- SiteSetting.stubs(:admin_site_traffic_event_cap).returns(2)+ SiteSetting.admin_site_traffic_event_cap = 2get "/admin/dashboard/site-traffic-explorer.json",params: request_params.merge(start_date: "2026-01-01")diff --git a/spec/services/admin_dashboard_site_traffic_explorer_spec.rb b/spec/services/admin_dashboard_site_traffic_explorer_spec.rbindex e760f2f835f..702352348ff 100644--- a/spec/services/admin_dashboard_site_traffic_explorer_spec.rb+++ b/spec/services/admin_dashboard_site_traffic_explorer_spec.rb@@ -170,7 +170,7 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do)end- it "runs one read-only analytics statement with a ten-second deadline" do+ it "runs one analytics statement with a ten-second deadline" doquery_row = {"pageview_limited" => false,"summary" => {@@ -183,7 +183,6 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do"network" => nil,},}- DB.expects(:exec).with("SET TRANSACTION READ ONLY").onceDB.expects(:exec).with("SET LOCAL statement_timeout = 10000").onceDB.expects(:query_hash).once.returns([query_row])@@ -235,7 +234,7 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do).to eq([12, 11, [{ value: "", label: "Direct / unknown", pageviews: 11 }]])end- it "groups session entries by normalized referrer" do+ it "groups acquisition entries by normalized referrer" doFabricate(:browser_pageview_event,normalized_referrer: "external.example?article=traffic",@@ -258,17 +257,27 @@ RSpec.describe AdminDashboardSiteTrafficExplorer dolabel: "external.example?article=traffic",pageviews: 1,},- {- value: "test.localhost?view=latest",- label: "test.localhost?view=latest",- pageviews: 1,- },],)end+ it "does not report partial data when the population exactly matches the cap" do+ SiteSetting.admin_site_traffic_event_cap = 11++ expect(result.traffic.slice(:partial_data, :summary)).to eq(+ partial_data: nil,+ summary: {+ "pageviews" => 11,+ "distinct_sessions" => 11,+ "logged_in_share" => 0,+ "bounce_rate" => 100,+ "average_session_duration_seconds" => 0,+ },+ )+ end+it "does not promote a capped session continuation to an entry" do- SiteSetting.stubs(:admin_site_traffic_event_cap).returns(1)+ SiteSetting.admin_site_traffic_event_cap = 1Fabricate(:browser_pageview_event,url: "/capped-session-entry",@@ -298,18 +307,20 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do).to eq([1, [], [], "pageview_limit"])end- it "uses the earliest event id to break a session entry timestamp tie" do+ it "uses referrers rather than event ids to identify acquisition entries" docreated_at = Time.zone.local(2026, 5, 11, 10)Fabricate(:browser_pageview_event,- url: "/first-at-timestamp",+ url: "/acquisition-at-timestamp",+ normalized_referrer: "external.example/path",session_id: "same-timestamp",source: BrowserPageviewEvent::SOURCE_BEACON,created_at:,)Fabricate(:browser_pageview_event,- url: "/second-at-timestamp",+ url: "/internal-at-timestamp",+ normalized_referrer: "test.localhost/acquisition-at-timestamp",session_id: "same-timestamp",source: BrowserPageviewEvent::SOURCE_BEACON,created_at:,@@ -318,7 +329,13 @@ RSpec.describe AdminDashboardSiteTrafficExplorer doentry_urls = result.traffic.dig(:dimensions, "entry_urls")expect(entry_urls.select { |row| row[:value].end_with?("-at-timestamp") }).to eq(- [{ value: "/first-at-timestamp", label: "/first-at-timestamp", pageviews: 1 }],+ [+ {+ value: "/acquisition-at-timestamp",+ label: "/acquisition-at-timestamp",+ pageviews: 1,+ },+ ],)endenddiff --git a/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb b/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rbindex 7e720d14339..c3b975bd34a 100644--- a/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb+++ b/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb@@ -405,7 +405,7 @@ RSpec.describe "Admin Dashboard Redesign | Site Traffic Explorer" doit "warns an admin when the selected range has incomplete traffic data",time: Time.zone.local(2026, 5, 14, 12, 0, 0) dosign_in(admin)- SiteSetting.stubs(:admin_site_traffic_event_cap).returns(2)+ SiteSetting.admin_site_traffic_event_cap = 2[["/first-retained", "2026-02-15 09:00:00"],