Skip to content

A platform for community discussion. Free, open, simple.

PERF: Optimize site traffic explorer SQL query (#42547)

Alan Guo Xiang Tan2b6ae5fa0b0c
diff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rb
index d62153ac9c9..be71fdf1e42 100644
--- a/app/models/browser_pageview_event.rb
+++ b/app/models/browser_pageview_event.rb
@@ -216,10 +216,12 @@ class BrowserPageviewEvent < ActiveRecord::Base
RETENTION_PERIOD.ago.beginning_of_day
end
- def self.rollup_source_condition(table: nil)
+ def self.rollup_source_condition(table: nil, start_date: nil, end_date: nil)
prefix = table ? "#{table}." : ""
cutover_date = beacon_cutover_date
return "#{prefix}source = #{SOURCE_PIGGYBACK}" if cutover_date.nil?
+ return "#{prefix}source = #{SOURCE_PIGGYBACK}" if end_date && end_date.to_date <= cutover_date
+ return "#{prefix}source = #{SOURCE_BEACON}" if start_date && start_date.to_date >= cutover_date
sanitize_sql_array(
[
@@ -273,6 +275,7 @@ end
#
# Indexes
#
+# idx_bpe_beacon_created_at_id (created_at,id) WHERE (source = 2)
# idx_bpe_created_at_country_code (created_at,country_code)
# idx_bpe_created_at_normalized_referrer (created_at,normalized_referrer)
# idx_bpe_ip_ua_created_at (ip_address,user_agent,created_at)
diff --git a/app/services/admin_dashboard_site_traffic_explorer.rb b/app/services/admin_dashboard_site_traffic_explorer.rb
index ac3d0827f19..15d9120c737 100644
--- a/app/services/admin_dashboard_site_traffic_explorer.rb
+++ b/app/services/admin_dashboard_site_traffic_explorer.rb
@@ -118,10 +118,14 @@ class AdminDashboardSiteTrafficExplorer
end
def execute_query(start_date:, end_date:, filters:)
+ parameters = query_params(start_date:, end_date:, filters:)
+
ActiveRecord::Base.transaction(requires_new: true) do
- DB.exec("SET TRANSACTION READ ONLY")
DB.exec("SET LOCAL statement_timeout = #{STATEMENT_TIMEOUT_MS}")
- DB.query_hash(query, query_params(start_date:, end_date:, filters:)).first
+ DB.query_hash(
+ query(start_date: parameters[:start_date], end_date: parameters[:end_date]),
+ parameters,
+ ).first
end
end
@@ -132,10 +136,9 @@ class AdminDashboardSiteTrafficExplorer
{
start_date: effective_start_date,
- retention_cutoff: retention_cutoff,
end_date: end_date + 1.day,
cap: cap,
- cap_plus_one: cap + 1,
+ site_host: BrowserPageviewEventUrlNormalizer.normalize_host(Discourse.current_hostname),
top_url: filters[:top_url],
entry_url: filters[:entry_url],
referrer: filters[:referrer],
@@ -155,78 +158,48 @@ class AdminDashboardSiteTrafficExplorer
}
end
- def query
+ def query(start_date:, end_date:)
+ source_condition =
+ BrowserPageviewEvent.rollup_source_condition(table: "bpe", start_date:, end_date:)
+
<<~SQL
- WITH eligible AS MATERIALIZED (
+ WITH population AS MATERIALIZED (
SELECT
- candidates.id,
- candidates.created_at,
- candidates.session_id,
- candidates.user_id,
- candidates.normalized_url,
- candidates.country_code,
- candidates.asn,
- candidates.ip_address,
- candidates.user_agent,
- candidates.score,
- ROW_NUMBER() OVER (
- ORDER BY candidates.created_at DESC, candidates.id DESC
- ) AS cap_position
- FROM (
- SELECT
- bpe.id,
- bpe.created_at,
- bpe.session_id,
- bpe.user_id,
- bpe.normalized_url,
- bpe.country_code,
- bpe.asn,
- bpe.ip_address,
- bpe.user_agent,
- bpe.score
- FROM browser_pageview_events bpe
- WHERE bpe.created_at >= :start_date
- AND bpe.created_at < :end_date
- AND #{BrowserPageviewEvent.rollup_source_condition(table: "bpe")}
- ORDER BY bpe.created_at DESC, bpe.id DESC
- LIMIT :cap_plus_one
- ) candidates
+ bpe.id,
+ bpe.created_at,
+ bpe.session_id,
+ bpe.user_id,
+ bpe.normalized_url,
+ bpe.normalized_referrer,
+ bpe.country_code,
+ bpe.asn,
+ bpe.ip_address,
+ bpe.user_agent,
+ bpe.score
+ FROM browser_pageview_events bpe
+ WHERE bpe.created_at >= :start_date
+ AND bpe.created_at < :end_date
+ AND #{source_condition}
+ ORDER BY bpe.created_at DESC, bpe.id DESC
+ LIMIT :cap
),
- population AS NOT MATERIALIZED (
+ population_stats AS MATERIALIZED (
SELECT
- id,
- created_at,
- session_id,
- user_id,
- normalized_url,
- country_code,
- asn,
- ip_address,
- user_agent,
- score
- FROM eligible
- WHERE cap_position <= :cap
- ),
- population_sessions AS (
- SELECT DISTINCT session_id
+ COUNT(*) AS row_count,
+ MIN(created_at) AS oldest_created_at
FROM population
),
- session_entries AS (
- SELECT entry.*
- FROM population_sessions
- CROSS JOIN LATERAL (
- SELECT
- bpe.id,
- bpe.session_id,
- bpe.normalized_url,
- bpe.normalized_referrer
- FROM browser_pageview_events bpe
- WHERE bpe.session_id = population_sessions.session_id
- AND bpe.created_at >= :retention_cutoff
- AND #{BrowserPageviewEvent.rollup_source_condition(table: "bpe")}
- ORDER BY bpe.created_at, bpe.id
- LIMIT 1
- ) entry
+ population_boundary AS MATERIALIZED (
+ SELECT
+ population_stats.row_count,
+ population_stats.oldest_created_at,
+ MIN(population.id) AS oldest_id
+ FROM population_stats
+ LEFT JOIN population
+ ON population.created_at = population_stats.oldest_created_at
+ GROUP BY
+ population_stats.row_count,
+ population_stats.oldest_created_at
),
browser_values AS MATERIALIZED (
SELECT
@@ -244,15 +217,23 @@ class AdminDashboardSiteTrafficExplorer
),
classified AS (
SELECT
- population.id,
population.created_at,
population.session_id,
population.user_id,
population.normalized_url,
+ population.normalized_referrer,
population.country_code,
population.asn,
population.ip_address,
browser_values.browser,
+ (
+ population.normalized_referrer IS NULL
+ OR split_part(
+ split_part(population.normalized_referrer, '/', 1),
+ '?',
+ 1
+ ) <> :site_host
+ ) AS acquisition_entry,
(
:crawler_detection_enabled
AND COALESCE(population.score, 0) > :crawler_threshold
@@ -263,17 +244,23 @@ class AdminDashboardSiteTrafficExplorer
),
dimensioned AS (
SELECT
- classified.*,
+ classified.created_at,
+ classified.session_id,
+ classified.user_id,
+ classified.normalized_url,
+ classified.country_code,
+ classified.asn,
+ classified.ip_address,
+ classified.browser,
+ classified.likely_crawler,
CASE
- WHEN classified.id = session_entries.id THEN session_entries.normalized_url
+ WHEN classified.acquisition_entry THEN classified.normalized_url
END AS entry_url,
CASE
- WHEN session_entries.id IS NULL OR classified.id <> session_entries.id THEN NULL
- ELSE COALESCE(session_entries.normalized_referrer, '')
+ WHEN classified.acquisition_entry
+ THEN COALESCE(classified.normalized_referrer, '')
END AS referrer
FROM classified
- LEFT JOIN session_entries
- ON session_entries.session_id = classified.session_id
),
filtered AS MATERIALIZED (
SELECT *
@@ -326,16 +313,6 @@ class AdminDashboardSiteTrafficExplorer
LEFT JOIN browser_pageview_session_engagements engagement
ON engagement.session_id = sessions.session_id
),
- traffic_summary AS (
- SELECT
- COUNT(*) FILTER (
- WHERE :traffic_type_filtered OR NOT likely_crawler
- )::integer AS pageviews,
- COUNT(*) FILTER (
- WHERE NOT likely_crawler AND user_id IS NOT NULL
- )::integer AS logged_in_pageviews
- FROM filtered
- ),
series_rows AS (
SELECT
created_at::date AS date,
@@ -351,18 +328,53 @@ class AdminDashboardSiteTrafficExplorer
COUNT(*) FILTER (WHERE likely_crawler)::integer AS likely_crawler_pageviews
FROM filtered
GROUP BY created_at::date
+ ),
+ traffic_summary AS (
+ SELECT
+ COALESCE(SUM(pageviews), 0)::integer AS pageviews,
+ COALESCE(SUM(logged_in_human_pageviews), 0)::integer AS logged_in_pageviews
+ FROM series_rows
+ ),
+ bounded_dimension_rows AS MATERIALIZED (
+ SELECT
+ CASE
+ WHEN GROUPING(country_code) = 0 THEN 'countries'
+ WHEN GROUPING(browser) = 0 THEN 'browsers'
+ END AS dimension,
+ country_code,
+ browser,
+ COUNT(*)::integer AS pageviews,
+ MIN(ip_address) AS representative_ip
+ FROM filtered
+ GROUP BY GROUPING SETS ((country_code), (browser))
)
SELECT
- (SELECT EXISTS (SELECT 1 FROM eligible WHERE cap_position > :cap)) AS pageview_limited,
+ (
+ SELECT
+ population_boundary.row_count = :cap
+ AND EXISTS (
+ SELECT 1
+ FROM browser_pageview_events bpe
+ WHERE bpe.created_at >= :start_date
+ AND bpe.created_at < :end_date
+ AND #{source_condition}
+ AND (bpe.created_at, bpe.id) < (
+ population_boundary.oldest_created_at,
+ population_boundary.oldest_id
+ )
+ LIMIT 1
+ )
+ FROM population_boundary
+ ) AS pageview_limited,
jsonb_build_object(
'country', (
- SELECT MIN(host(ip_address))
+ SELECT host(MIN(ip_address))
FROM population
WHERE :country IS NOT NULL
AND country_code = :country
),
'network', (
- SELECT MIN(host(ip_address))
+ SELECT host(MIN(ip_address))
FROM population
WHERE :network_asn IS NOT NULL
AND asn = :network_asn
@@ -476,11 +488,11 @@ class AdminDashboardSiteTrafficExplorer
FROM (
SELECT
country_code AS value,
- COUNT(*)::integer AS pageviews,
- MIN(host(ip_address)) AS representative_ip
- FROM filtered
- WHERE country_code IS NOT NULL
- GROUP BY country_code
+ pageviews,
+ host(representative_ip) AS representative_ip
+ FROM bounded_dimension_rows
+ WHERE dimension = 'countries'
+ AND country_code IS NOT NULL
ORDER BY pageviews DESC, value
LIMIT :dimension_limit
) rows
@@ -501,7 +513,7 @@ class AdminDashboardSiteTrafficExplorer
SELECT
'AS' || asn AS value,
COUNT(*)::integer AS pageviews,
- MIN(host(ip_address)) AS representative_ip
+ host(MIN(ip_address)) AS representative_ip
FROM filtered
WHERE asn IS NOT NULL
GROUP BY asn
@@ -518,9 +530,9 @@ class AdminDashboardSiteTrafficExplorer
'[]'::jsonb
)
FROM (
- SELECT browser AS value, COUNT(*)::integer AS pageviews
- FROM filtered
- GROUP BY browser
+ SELECT browser AS value, pageviews
+ FROM bounded_dimension_rows
+ WHERE dimension = 'browsers'
ORDER BY pageviews DESC, value
LIMIT :dimension_limit
) rows
diff --git a/db/migrate/20260812094609_add_beacon_traffic_explorer_index_to_browser_pageview_events.rb b/db/migrate/20260812094609_add_beacon_traffic_explorer_index_to_browser_pageview_events.rb
new file mode 100644
index 00000000000..43a51eb258f
--- /dev/null
+++ b/db/migrate/20260812094609_add_beacon_traffic_explorer_index_to_browser_pageview_events.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+class AddBeaconTrafficExplorerIndexToBrowserPageviewEvents < ActiveRecord::Migration[8.0]
+ disable_ddl_transaction!
+
+ INDEX_NAME = "idx_bpe_beacon_created_at_id"
+ private_constant :INDEX_NAME
+
+ def up
+ remove_index :browser_pageview_events,
+ name: INDEX_NAME,
+ algorithm: :concurrently,
+ if_exists: true
+ add_index :browser_pageview_events,
+ %i[created_at id],
+ name: INDEX_NAME,
+ order: {
+ created_at: :desc,
+ id: :desc,
+ },
+ where: "source = 2",
+ algorithm: :concurrently
+ end
+
+ def down
+ remove_index :browser_pageview_events,
+ name: INDEX_NAME,
+ algorithm: :concurrently,
+ if_exists: true
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index 6d1cecf1f6f..0e07dbaa594 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -17584,6 +17584,13 @@ CREATE UNIQUE INDEX idx_bpcd_rollups_date_country_unique ON public.browser_pagev
CREATE UNIQUE INDEX idx_bpcrawler_rollups_date_logged_in_unique ON public.browser_pageview_crawler_daily_rollups USING btree (date, logged_in);
+--
+-- Name: idx_bpe_beacon_created_at_id; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_bpe_beacon_created_at_id ON public.browser_pageview_events USING btree (created_at DESC, id DESC) WHERE (source = 2);
+
+
--
-- Name: idx_bpe_created_at_country_code; Type: INDEX; Schema: public; Owner: -
--
@@ -23151,6 +23158,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groups
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
+('20260812094609'),
('20260810154331'),
('20260810012238'),
('20260807182856'),
diff --git a/spec/models/browser_pageview_event_spec.rb b/spec/models/browser_pageview_event_spec.rb
index 40e4542c6cf..f3eff9a23be 100644
--- a/spec/models/browser_pageview_event_spec.rb
+++ b/spec/models/browser_pageview_event_spec.rb
@@ -99,6 +99,16 @@ RSpec.describe BrowserPageviewEvent do
piggyback = Fabricate(:browser_pageview_event, source: :piggyback)
Fabricate(:browser_pageview_event, source: :beacon)
+ bounded_condition =
+ described_class.rollup_source_condition(
+ start_date: Date.new(2026, 6, 1),
+ end_date: Date.new(2026, 6, 20),
+ )
+
+ expect(described_class.rollup_source_condition).to eq(
+ "source = #{described_class::SOURCE_PIGGYBACK}",
+ )
+ expect(bounded_condition).to eq("source = #{described_class::SOURCE_PIGGYBACK}")
expect(described_class.where(described_class.rollup_source_condition)).to contain_exactly(
piggyback,
)
@@ -131,6 +141,45 @@ RSpec.describe BrowserPageviewEvent do
expect(condition).to eq("e.source = #{described_class::SOURCE_PIGGYBACK}")
end
+
+ it "matches only piggyback events when the bounded range ends at the cutover" do
+ described_class.stubs(:beacon_cutover_date).returns(Date.new(2026, 6, 10))
+
+ condition =
+ described_class.rollup_source_condition(
+ start_date: Date.new(2026, 6, 1),
+ end_date: Date.new(2026, 6, 10),
+ )
+
+ expect(condition).to eq("source = #{described_class::SOURCE_PIGGYBACK}")
+ end
+
+ it "matches only beacon events when the bounded range starts at the cutover" do
+ described_class.stubs(:beacon_cutover_date).returns(Date.new(2026, 6, 10))
+
+ condition =
+ described_class.rollup_source_condition(
+ start_date: Date.new(2026, 6, 10),
+ end_date: Date.new(2026, 6, 20),
+ )
+
+ expect(condition).to eq("source = #{described_class::SOURCE_BEACON}")
+ end
+
+ it "keeps the mixed source condition when the bounded range crosses the cutover" do
+ described_class.stubs(:beacon_cutover_date).returns(Date.new(2026, 6, 10))
+
+ condition =
+ described_class.rollup_source_condition(
+ start_date: Date.new(2026, 6, 1),
+ end_date: Date.new(2026, 6, 20),
+ )
+
+ expect(condition).to include(
+ "created_at < '2026-06-10' AND source = #{described_class::SOURCE_PIGGYBACK}",
+ "created_at >= '2026-06-10' AND source = #{described_class::SOURCE_BEACON}",
+ )
+ end
end
it "truncates string fields before saving" do
diff --git a/spec/requests/admin/dashboard_controller_spec.rb b/spec/requests/admin/dashboard_controller_spec.rb
index 7c45faad306..18fea810156 100644
--- a/spec/requests/admin/dashboard_controller_spec.rb
+++ b/spec/requests/admin/dashboard_controller_spec.rb
@@ -1432,7 +1432,7 @@ RSpec.describe Admin::DashboardController do
user_agent: "Mozilla/5.0 AppleWebKit/537.36 Chrome/124.0.0.0 Safari/537.36",
user_id: admin.id,
session_id: "admin-session",
- normalized_referrer: "ignored.example/return?from=latest",
+ normalized_referrer: "test.localhost/landing",
normalized_referrer_version: BrowserPageviewEventUrlNormalizer::REFERRER_VERSION,
source: BrowserPageviewEvent::SOURCE_BEACON,
created_at: "2026-05-10 10:01:00",
@@ -1584,6 +1584,38 @@ RSpec.describe Admin::DashboardController do
},
)
end
+
+ it "treats same-site referrers as internal navigation" do
+ same_site_pageview =
+ Fabricate(
+ :browser_pageview_event,
+ url: "https://test.localhost/same-site-full-load",
+ normalized_referrer: "test.localhost/previous-page",
+ session_id: "same-site-full-load",
+ source: BrowserPageviewEvent::SOURCE_BEACON,
+ created_at: "2026-05-11 11:00:00",
+ )
+
+ get "/admin/dashboard/site-traffic-explorer.json", params: request_params
+
+ dimensions = response.parsed_body.fetch("dimensions")
+ top_url =
+ dimensions
+ .fetch("top_urls")
+ .find { |row| row["value"] == same_site_pageview.normalized_url }
+ entry_url =
+ dimensions
+ .fetch("entry_urls")
+ .find { |row| row["value"] == same_site_pageview.normalized_url }
+ referrer =
+ dimensions
+ .fetch("referrers")
+ .find { |row| row["value"] == same_site_pageview.normalized_referrer }
+
+ expect([response.status, top_url&.fetch("pageviews"), entry_url, referrer]).to eq(
+ [200, 1, nil, nil],
+ )
+ end
end
context "when the selected date range exceeds retention" do
@@ -1720,7 +1752,7 @@ RSpec.describe Admin::DashboardController do
before do
sign_in(admin)
- SiteSetting.stubs(:admin_site_traffic_event_cap).returns(2)
+ SiteSetting.admin_site_traffic_event_cap = 2
end
it "returns no more than the configured pageview cap" do
@@ -1815,7 +1847,7 @@ RSpec.describe Admin::DashboardController do
it "returns both partial-data reasons" do
sign_in(admin)
- SiteSetting.stubs(:admin_site_traffic_event_cap).returns(2)
+ SiteSetting.admin_site_traffic_event_cap = 2
get "/admin/dashboard/site-traffic-explorer.json",
params: request_params.merge(start_date: "2026-01-01")
diff --git a/spec/services/admin_dashboard_site_traffic_explorer_spec.rb b/spec/services/admin_dashboard_site_traffic_explorer_spec.rb
index e760f2f835f..702352348ff 100644
--- a/spec/services/admin_dashboard_site_traffic_explorer_spec.rb
+++ b/spec/services/admin_dashboard_site_traffic_explorer_spec.rb
@@ -170,7 +170,7 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do
)
end
- it "runs one read-only analytics statement with a ten-second deadline" do
+ it "runs one analytics statement with a ten-second deadline" do
query_row = {
"pageview_limited" => false,
"summary" => {
@@ -183,7 +183,6 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do
"network" => nil,
},
}
- DB.expects(:exec).with("SET TRANSACTION READ ONLY").once
DB.expects(:exec).with("SET LOCAL statement_timeout = 10000").once
DB.expects(:query_hash).once.returns([query_row])
@@ -235,7 +234,7 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do
).to eq([12, 11, [{ value: "", label: "Direct / unknown", pageviews: 11 }]])
end
- it "groups session entries by normalized referrer" do
+ it "groups acquisition entries by normalized referrer" do
Fabricate(
:browser_pageview_event,
normalized_referrer: "external.example?article=traffic",
@@ -258,17 +257,27 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do
label: "external.example?article=traffic",
pageviews: 1,
},
- {
- value: "test.localhost?view=latest",
- label: "test.localhost?view=latest",
- pageviews: 1,
- },
],
)
end
+ it "does not report partial data when the population exactly matches the cap" do
+ SiteSetting.admin_site_traffic_event_cap = 11
+
+ expect(result.traffic.slice(:partial_data, :summary)).to eq(
+ partial_data: nil,
+ summary: {
+ "pageviews" => 11,
+ "distinct_sessions" => 11,
+ "logged_in_share" => 0,
+ "bounce_rate" => 100,
+ "average_session_duration_seconds" => 0,
+ },
+ )
+ end
+
it "does not promote a capped session continuation to an entry" do
- SiteSetting.stubs(:admin_site_traffic_event_cap).returns(1)
+ SiteSetting.admin_site_traffic_event_cap = 1
Fabricate(
:browser_pageview_event,
url: "/capped-session-entry",
@@ -298,18 +307,20 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do
).to eq([1, [], [], "pageview_limit"])
end
- it "uses the earliest event id to break a session entry timestamp tie" do
+ it "uses referrers rather than event ids to identify acquisition entries" do
created_at = Time.zone.local(2026, 5, 11, 10)
Fabricate(
:browser_pageview_event,
- url: "/first-at-timestamp",
+ url: "/acquisition-at-timestamp",
+ normalized_referrer: "external.example/path",
session_id: "same-timestamp",
source: BrowserPageviewEvent::SOURCE_BEACON,
created_at:,
)
Fabricate(
:browser_pageview_event,
- url: "/second-at-timestamp",
+ url: "/internal-at-timestamp",
+ normalized_referrer: "test.localhost/acquisition-at-timestamp",
session_id: "same-timestamp",
source: BrowserPageviewEvent::SOURCE_BEACON,
created_at:,
@@ -318,7 +329,13 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do
entry_urls = result.traffic.dig(:dimensions, "entry_urls")
expect(entry_urls.select { |row| row[:value].end_with?("-at-timestamp") }).to eq(
- [{ value: "/first-at-timestamp", label: "/first-at-timestamp", pageviews: 1 }],
+ [
+ {
+ value: "/acquisition-at-timestamp",
+ label: "/acquisition-at-timestamp",
+ pageviews: 1,
+ },
+ ],
)
end
end
diff --git a/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb b/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb
index 7e720d14339..c3b975bd34a 100644
--- a/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb
+++ b/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb
@@ -405,7 +405,7 @@ RSpec.describe "Admin Dashboard Redesign | Site Traffic Explorer" do
it "warns an admin when the selected range has incomplete traffic data",
time: Time.zone.local(2026, 5, 14, 12, 0, 0) do
sign_in(admin)
- SiteSetting.stubs(:admin_site_traffic_event_cap).returns(2)
+ SiteSetting.admin_site_traffic_event_cap = 2
[
["/first-retained", "2026-02-15 09:00:00"],