Skip to content

A platform for community discussion. Free, open, simple.

FIX: Select the browser pageview counters for each day (#43766)

Krzysztof Kotlarekda28a4cf4b2b
diff --git a/app/models/application_request.rb b/app/models/application_request.rb
index f3ff8eb0f84..0e9283a6d9f 100644
--- a/app/models/application_request.rb
+++ b/app/models/application_request.rb
@@ -28,38 +28,41 @@ class ApplicationRequest < ActiveRecord::Base
include CachedCounting
+ BROWSER_PAGEVIEW_TRANSPORTS = {
+ "page_view_anon_browser" => "page_view_anon_browser_beacon",
+ "page_view_logged_in_browser" => "page_view_logged_in_browser_beacon",
+ }.freeze
+
+ # Browser pageviews arrive through two transports: a beacon request and,
+ # historically, a piggybacked header on regular requests. Both can record the
+ # same visit, so counts are read from beacons whenever they recorded traffic
+ # and fall back to the piggyback counters when they did not. Anonymous and
+ # logged in visitors are decided separately, because either cohort can be the
+ # only one a beacon saw on a given day.
def self.browser_pageviews
- legacy_types = %i[page_view_anon_browser page_view_logged_in_browser]
- beacon_types = %i[page_view_anon_browser_beacon page_view_logged_in_browser_beacon]
- cutover_date = browser_pageview_beacon_cutover_date
-
- return where(req_type: legacy_types) if cutover_date.nil?
+ conditions =
+ BROWSER_PAGEVIEW_TRANSPORTS.map do |legacy_type, beacon_type|
+ legacy_id, beacon_id = req_types.values_at(legacy_type, beacon_type)
+
+ <<~SQL
+ (
+ application_requests.req_type IN (#{legacy_id}, #{beacon_id})
+ AND (application_requests.req_type = #{beacon_id}) = EXISTS (
+ SELECT 1
+ FROM application_requests beacons
+ WHERE beacons.date = application_requests.date
+ AND beacons.req_type = #{beacon_id}
+ AND beacons.count > 0
+ )
+ )
+ SQL
+ end
- # Both transports can record the same visit, so select one source for each date.
- where(req_type: legacy_types).where("date < ?", cutover_date).or(
- where(req_type: beacon_types).where("date >= ?", cutover_date),
+ where(req_type: req_types.values_at(*BROWSER_PAGEVIEW_TRANSPORTS.to_a.flatten)).where(
+ conditions.join(" OR "),
)
end
- def self.browser_pageview_beacon_cutover_date
- first_beacon_date =
- where(req_type: %i[page_view_anon_browser_beacon page_view_logged_in_browser_beacon]).where(
- "count > 0",
- ).minimum(:date)
- return if first_beacon_date.nil?
-
- # The first beacon day may be partial. Prefer existing piggyback traffic for that
- # whole day, but retain the first day on sites that only recorded beacons.
- if where(
- date: first_beacon_date,
- req_type: %i[page_view_anon_browser page_view_logged_in_browser],
- ).where("count > 0").exists?
- first_beacon_date.next_day
- else
- first_beacon_date
- end
- end
-
def self.browser_pageview_count_for_period(type, since)
browser_pageviews
.where(req_type: [type, "#{type}_beacon"])
diff --git a/app/services/admin_dashboard_site_traffic.rb b/app/services/admin_dashboard_site_traffic.rb
index 39a740d29e8..7229f155911 100644
--- a/app/services/admin_dashboard_site_traffic.rb
+++ b/app/services/admin_dashboard_site_traffic.rb
@@ -307,16 +307,32 @@ class AdminDashboardSiteTraffic
}
end
- def beacon_cutover_date
- return if SiteSetting.use_legacy_pageviews
- return @beacon_cutover_date if defined?(@beacon_cutover_date)
+ def human_pageviews_sql(range_start_date, range_end_date)
+ requests =
+ if SiteSetting.use_legacy_pageviews
+ ApplicationRequest.where(req_type: selected_request_types.values_at(:logged_in, :anonymous))
+ else
+ ApplicationRequest.browser_pageviews
+ end
- @beacon_cutover_date = ApplicationRequest.browser_pageview_beacon_cutover_date
+ <<~SQL
+ SELECT date,
+ COALESCE(SUM(count) FILTER (WHERE req_type IN (#{logged_in_req_types.join(",")})), 0)::bigint AS logged_in,
+ COALESCE(SUM(count) FILTER (WHERE req_type IN (#{anonymous_req_types.join(",")})), 0)::bigint AS anonymous
+ FROM (#{requests.where(date: range_start_date..range_end_date).to_sql}) requests
+ GROUP BY date
+ SQL
end
- def traffic_rows(range_start_date, range_end_date)
- cutover_date = beacon_cutover_date || (range_end_date + 1.day)
+ def logged_in_req_types
+ [selected_request_types[:logged_in], beacon_request_types[:logged_in]].compact
+ end
+ def anonymous_req_types
+ [selected_request_types[:anonymous], beacon_request_types[:anonymous]].compact
+ end
+
+ def traffic_rows(range_start_date, range_end_date)
DB.query(
<<~SQL,
WITH dates AS (
@@ -338,34 +354,19 @@ class AdminDashboardSiteTraffic
AND date >= CAST(:start_date AS date)
AND date <= CAST(:end_date AS date)
GROUP BY date
+ ),
+ human_pageviews AS (
+ #{human_pageviews_sql(range_start_date, range_end_date)}
)
SELECT
dates.date,
GREATEST(
0,
- COALESCE(
- SUM(
- CASE
- WHEN dates.date < :beacon_cutover_date AND ar.req_type = :logged_in_req_type THEN ar.count
- WHEN dates.date >= :beacon_cutover_date AND ar.req_type = :logged_in_beacon_req_type THEN ar.count
- ELSE 0
- END
- ),
- 0
- ) - COALESCE(MAX(lc.logged_in), 0)
+ COALESCE(MAX(hp.logged_in), 0) - COALESCE(MAX(lc.logged_in), 0)
)::bigint AS logged_in,
GREATEST(
0,
- COALESCE(
- SUM(
- CASE
- WHEN dates.date < :beacon_cutover_date AND ar.req_type = :anonymous_req_type THEN ar.count
- WHEN dates.date >= :beacon_cutover_date AND ar.req_type = :anonymous_beacon_req_type THEN ar.count
- ELSE 0
- END
- ),
- 0
- ) - COALESCE(MAX(lc.anonymous), 0)
+ COALESCE(MAX(hp.anonymous), 0) - COALESCE(MAX(lc.anonymous), 0)
)::bigint AS anonymous,
COALESCE(SUM(CASE WHEN ar.req_type = :crawler_req_type THEN ar.count ELSE 0 END), 0)::bigint AS crawlers,
COALESCE(SUM(CASE WHEN ar.req_type = :embedded_req_type THEN ar.count ELSE 0 END), 0)::bigint AS embedded,
@@ -376,14 +377,8 @@ class AdminDashboardSiteTraffic
FROM dates
LEFT JOIN application_requests ar
ON ar.date = dates.date
- AND ar.req_type IN (
- :logged_in_req_type,
- :anonymous_req_type,
- :logged_in_beacon_req_type,
- :anonymous_beacon_req_type,
- :crawler_req_type,
- :embedded_req_type
- )
+ AND ar.req_type IN (:crawler_req_type, :embedded_req_type)
+ LEFT JOIN human_pageviews hp ON hp.date = dates.date
LEFT JOIN likely_crawlers lc ON lc.date = dates.date
GROUP BY dates.date
ORDER BY dates.date ASC
@@ -392,11 +387,6 @@ class AdminDashboardSiteTraffic
end_date: range_end_date,
likely_crawlers_enabled: likely_crawlers_enabled?,
login_required: login_required?,
- beacon_cutover_date: cutover_date,
- logged_in_req_type: selected_request_types[:logged_in],
- anonymous_req_type: selected_request_types[:anonymous],
- logged_in_beacon_req_type: beacon_request_types[:logged_in],
- anonymous_beacon_req_type: beacon_request_types[:anonymous],
crawler_req_type: selected_request_types[:crawlers],
embedded_req_type: selected_request_types[:embedded],
)
diff --git a/spec/lib/statistics_spec.rb b/spec/lib/statistics_spec.rb
index aaea0ab4843..bc96a220631 100644
--- a/spec/lib/statistics_spec.rb
+++ b/spec/lib/statistics_spec.rb
@@ -212,7 +212,7 @@ RSpec.describe Statistics do
after { ApplicationRequest.disable }
- it "preserves piggyback visitor estimates on the first partial beacon day" do
+ it "estimates visitors from beacon counters once beacons record traffic" do
ApplicationRequest.create!(
date: date.to_date,
req_type: :page_view_anon_browser_beacon,
@@ -225,7 +225,7 @@ RSpec.describe Statistics do
)
freeze_time(date) do
- expect(described_class.visitors).to eq(last_day: 3, "7_days": 6, "30_days": 9)
+ expect(described_class.visitors).to eq(last_day: 5, "7_days": 8, "30_days": 11)
end
end
@@ -301,7 +301,7 @@ RSpec.describe Statistics do
after { ApplicationRequest.disable }
- it "preserves piggyback EU visitor estimates on the first partial beacon day" do
+ it "estimates EU visitors from beacon counters once beacons record traffic" do
ApplicationRequest.create!(
date: date.to_date,
req_type: :page_view_anon_browser_beacon,
@@ -314,7 +314,7 @@ RSpec.describe Statistics do
)
freeze_time(date) do
- expect(described_class.eu_visitors).to eq(last_day: 3, "7_days": 3, "30_days": 5)
+ expect(described_class.eu_visitors).to eq(last_day: 5, "7_days": 4, "30_days": 6)
end
end
diff --git a/spec/models/application_request_spec.rb b/spec/models/application_request_spec.rb
index 9c3bf1b02c7..89a7ec743ce 100644
--- a/spec/models/application_request_spec.rb
+++ b/spec/models/application_request_spec.rb
@@ -17,29 +17,38 @@ RSpec.describe ApplicationRequest do
end
describe ".browser_pageviews" do
- it "preserves the first partial beacon day and switches both cohorts the following day" do
+ it "reads days with beacon traffic from beacons and the rest from piggyback counters" do
today = Date.current
history =
described_class.create!(date: today - 2, req_type: :page_view_anon_browser, count: 4)
described_class.create!(date: today - 2, req_type: :page_view_anon_browser_beacon, count: 0)
- first_day =
- described_class.create!(date: today - 1, req_type: :page_view_anon_browser, count: 1000)
- described_class.create!(date: today - 1, req_type: :page_view_anon_browser_beacon, count: 50)
- described_class.create!(
- date: today - 1,
- req_type: :page_view_logged_in_browser_beacon,
- count: 5,
- )
- beacon =
- described_class.create!(date: today, req_type: :page_view_anon_browser_beacon, count: 8)
- described_class.create!(date: today, req_type: :page_view_anon_browser, count: 3)
+ anon_beacon =
+ described_class.create!(
+ date: today - 1,
+ req_type: :page_view_anon_browser_beacon,
+ count: 50,
+ )
+ logged_in_beacon =
+ described_class.create!(
+ date: today - 1,
+ req_type: :page_view_logged_in_browser_beacon,
+ count: 5,
+ )
+ described_class.create!(date: today - 1, req_type: :page_view_anon_browser, count: 1000)
described_class.create!(
date: today,
req_type: :page_view_anon_browser_mobile_beacon,
count: 2,
)
-
- expect(described_class.browser_pageviews).to contain_exactly(history, first_day, beacon)
+ mobile_only_day =
+ described_class.create!(date: today, req_type: :page_view_anon_browser, count: 3)
+
+ expect(described_class.browser_pageviews).to contain_exactly(
+ history,
+ anon_beacon,
+ logged_in_beacon,
+ mobile_only_day,
+ )
end
it "includes the first beacon day when piggyback counters are absent or zero" do
@@ -51,6 +60,25 @@ RSpec.describe ApplicationRequest do
expect(described_class.browser_pageviews).to contain_exactly(beacon)
end
+ it "keeps using piggyback counters on days a lone early beacon did not cover" do
+ today = Date.current
+ described_class.create!(date: today - 30, req_type: :page_view_anon_browser_beacon, count: 1)
+ history = described_class.create!(date: today, req_type: :page_view_anon_browser, count: 100)
+
+ expect(described_class.browser_pageviews.where(date: today)).to contain_exactly(history)
+ end
+
+ it "decides anonymous and logged in traffic separately on the same day" do
+ today = Date.current
+ logged_in =
+ described_class.create!(date: today, req_type: :page_view_logged_in_browser, count: 40)
+ described_class.create!(date: today, req_type: :page_view_anon_browser, count: 60)
+ anon_beacon =
+ described_class.create!(date: today, req_type: :page_view_anon_browser_beacon, count: 25)
+
+ expect(described_class.browser_pageviews).to contain_exactly(logged_in, anon_beacon)
+ end
+
it "uses piggyback history when no beacons have been recorded" do
history =
described_class.create!(date: Date.current, req_type: :page_view_anon_browser, count: 4)
diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rb
index 788db5c9c23..8714dc634d6 100644
--- a/spec/models/report_spec.rb
+++ b/spec/models/report_spec.rb
@@ -406,7 +406,7 @@ RSpec.describe Report do
let(:report) { Report.find("page_view_total_reqs") }
- it "combines historical piggyback totals with beacon totals without counting overlap" do
+ it "reads days with beacon traffic from beacons and the rest from piggyback counters" do
SiteSetting.use_legacy_pageviews = false
ApplicationRequest.create!(
date: Date.current - 2,
@@ -420,8 +420,8 @@ RSpec.describe Report do
count: 8,
)
- expect(report.data).to eq([{ x: Date.current - 2, y: 4 }, { x: Date.current, y: 3 }])
- expect(report.total).to eq(7)
+ expect(report.data).to eq([{ x: Date.current - 2, y: 4 }, { x: Date.current, y: 8 }])
+ expect(report.total).to eq(12)
end
context "with no data" do
@@ -1830,31 +1830,51 @@ RSpec.describe Report do
expect(series["page_view_other"]).to eq([{ x: Date.current, y: 2 }])
end
- it "uses historical piggyback counts before switching both browser series to beacons" do
+ it "keeps piggyback counts in the browser series after an isolated day of beacon data" do
+ ApplicationRequest.create!(
+ date: Date.current - 2,
+ req_type: :page_view_anon_browser_beacon,
+ count: 1,
+ )
+ ApplicationRequest.create!(date: Date.current, req_type: :page_view_anon_browser, count: 30)
+ ApplicationRequest.create!(date: Date.current, req_type: :page_view_anon, count: 30)
+
+ series = reports.data.to_h { |entry| [entry[:req], entry[:data]] }
+
+ expect(series["page_view_anon_browser"].last).to eq({ x: Date.current, y: 30 })
+ expect(series["page_view_other"].last).to eq({ x: Date.current, y: 0 })
+ end
+
+ it "reads days with beacon traffic from beacons and the rest from piggyback counters" do
ApplicationRequest.create!(
date: Date.current - 2,
req_type: :page_view_anon_browser,
count: 4,
)
- ApplicationRequest.create!(date: Date.current, req_type: :page_view_anon_browser, count: 3)
ApplicationRequest.create!(
- date: Date.current,
+ date: Date.current - 2,
req_type: :page_view_logged_in_browser,
count: 6,
)
+ ApplicationRequest.create!(date: Date.current, req_type: :page_view_anon_browser, count: 3)
ApplicationRequest.create!(
date: Date.current,
req_type: :page_view_anon_browser_beacon,
count: 8,
)
+ ApplicationRequest.create!(
+ date: Date.current,
+ req_type: :page_view_logged_in_browser_beacon,
+ count: 2,
+ )
series = reports.data.to_h { |entry| [entry[:req], entry[:data]] }
expect(series["page_view_anon_browser"]).to eq(
- [{ x: Date.current - 2, y: 4 }, { x: Date.current, y: 3 }],
+ [{ x: Date.current - 2, y: 4 }, { x: Date.current, y: 8 }],
)
expect(series["page_view_logged_in_browser"]).to eq(
- [{ x: Date.current - 2, y: 0 }, { x: Date.current, y: 6 }],
+ [{ x: Date.current - 2, y: 6 }, { x: Date.current, y: 2 }],
)
expect(series["page_view_other"]).to eq(
[{ x: Date.current - 2, y: 0 }, { x: Date.current, y: 0 }],
diff --git a/spec/services/admin_dashboard_site_traffic_spec.rb b/spec/services/admin_dashboard_site_traffic_spec.rb
index 25012786290..bf029b698be 100644
--- a/spec/services/admin_dashboard_site_traffic_spec.rb
+++ b/spec/services/admin_dashboard_site_traffic_spec.rb
@@ -189,20 +189,36 @@ RSpec.describe AdminDashboardSiteTraffic do
)
end
- it "switches to beacon counters after the first day with beacon data" do
+ it "reads days with beacon traffic from beacons and the rest from piggyback counters" do
Fabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 20)
Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-01", count: 15)
+ Fabricate(:logged_in_browser_application_request, date: "2026-05-02", count: 25)
+ Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-02", count: 30)
+ Fabricate(:logged_in_browser_application_request, date: "2026-05-03", count: 40)
+
+ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")
+
+ expect(traffic_series_data(response, :logged_in)).to eq(
+ [
+ traffic_point("2026-05-01", 15),
+ traffic_point("2026-05-02", 30),
+ traffic_point("2026-05-03", 40),
+ ],
+ )
+ end
+
+ it "keeps counting piggyback traffic after an isolated day of beacon data" do
+ Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-01", count: 1)
Fabricate(:logged_in_browser_application_request, date: "2026-05-02", count: 30)
- Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-02", count: 25)
Fabricate(:logged_in_browser_application_request, date: "2026-05-03", count: 40)
response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")
expect(traffic_series_data(response, :logged_in)).to eq(
[
- traffic_point("2026-05-01", 20),
- traffic_point("2026-05-02", 25),
- traffic_point("2026-05-03", 0),
+ traffic_point("2026-05-01", 1),
+ traffic_point("2026-05-02", 30),
+ traffic_point("2026-05-03", 40),
],
)
end