A platform for community discussion. Free, open, simple.
FIX: Select the browser pageview counters for each day (#43766)
Krzysztof Kotlarekda28a4cf4b2b
diff --git a/app/models/application_request.rb b/app/models/application_request.rbindex f3ff8eb0f84..0e9283a6d9f 100644--- a/app/models/application_request.rb+++ b/app/models/application_request.rb@@ -28,38 +28,41 @@ class ApplicationRequest < ActiveRecord::Baseinclude CachedCounting+ BROWSER_PAGEVIEW_TRANSPORTS = {+ "page_view_anon_browser" => "page_view_anon_browser_beacon",+ "page_view_logged_in_browser" => "page_view_logged_in_browser_beacon",+ }.freeze++ # Browser pageviews arrive through two transports: a beacon request and,+ # historically, a piggybacked header on regular requests. Both can record the+ # same visit, so counts are read from beacons whenever they recorded traffic+ # and fall back to the piggyback counters when they did not. Anonymous and+ # logged in visitors are decided separately, because either cohort can be the+ # only one a beacon saw on a given day.def self.browser_pageviews- legacy_types = %i[page_view_anon_browser page_view_logged_in_browser]- beacon_types = %i[page_view_anon_browser_beacon page_view_logged_in_browser_beacon]- cutover_date = browser_pageview_beacon_cutover_date-- return where(req_type: legacy_types) if cutover_date.nil?+ conditions =+ BROWSER_PAGEVIEW_TRANSPORTS.map do |legacy_type, beacon_type|+ legacy_id, beacon_id = req_types.values_at(legacy_type, beacon_type)++ <<~SQL+ (+ application_requests.req_type IN (#{legacy_id}, #{beacon_id})+ AND (application_requests.req_type = #{beacon_id}) = EXISTS (+ SELECT 1+ FROM application_requests beacons+ WHERE beacons.date = application_requests.date+ AND beacons.req_type = #{beacon_id}+ AND beacons.count > 0+ )+ )+ SQL+ end- # Both transports can record the same visit, so select one source for each date.- where(req_type: legacy_types).where("date < ?", cutover_date).or(- where(req_type: beacon_types).where("date >= ?", cutover_date),+ where(req_type: req_types.values_at(*BROWSER_PAGEVIEW_TRANSPORTS.to_a.flatten)).where(+ conditions.join(" OR "),)end- def self.browser_pageview_beacon_cutover_date- first_beacon_date =- where(req_type: %i[page_view_anon_browser_beacon page_view_logged_in_browser_beacon]).where(- "count > 0",- ).minimum(:date)- return if first_beacon_date.nil?-- # The first beacon day may be partial. Prefer existing piggyback traffic for that- # whole day, but retain the first day on sites that only recorded beacons.- if where(- date: first_beacon_date,- req_type: %i[page_view_anon_browser page_view_logged_in_browser],- ).where("count > 0").exists?- first_beacon_date.next_day- else- first_beacon_date- end- end-def self.browser_pageview_count_for_period(type, since)browser_pageviews.where(req_type: [type, "#{type}_beacon"])diff --git a/app/services/admin_dashboard_site_traffic.rb b/app/services/admin_dashboard_site_traffic.rbindex 39a740d29e8..7229f155911 100644--- a/app/services/admin_dashboard_site_traffic.rb+++ b/app/services/admin_dashboard_site_traffic.rb@@ -307,16 +307,32 @@ class AdminDashboardSiteTraffic}end- def beacon_cutover_date- return if SiteSetting.use_legacy_pageviews- return @beacon_cutover_date if defined?(@beacon_cutover_date)+ def human_pageviews_sql(range_start_date, range_end_date)+ requests =+ if SiteSetting.use_legacy_pageviews+ ApplicationRequest.where(req_type: selected_request_types.values_at(:logged_in, :anonymous))+ else+ ApplicationRequest.browser_pageviews+ end- @beacon_cutover_date = ApplicationRequest.browser_pageview_beacon_cutover_date+ <<~SQL+ SELECT date,+ COALESCE(SUM(count) FILTER (WHERE req_type IN (#{logged_in_req_types.join(",")})), 0)::bigint AS logged_in,+ COALESCE(SUM(count) FILTER (WHERE req_type IN (#{anonymous_req_types.join(",")})), 0)::bigint AS anonymous+ FROM (#{requests.where(date: range_start_date..range_end_date).to_sql}) requests+ GROUP BY date+ SQLend- def traffic_rows(range_start_date, range_end_date)- cutover_date = beacon_cutover_date || (range_end_date + 1.day)+ def logged_in_req_types+ [selected_request_types[:logged_in], beacon_request_types[:logged_in]].compact+ end+ def anonymous_req_types+ [selected_request_types[:anonymous], beacon_request_types[:anonymous]].compact+ end++ def traffic_rows(range_start_date, range_end_date)DB.query(<<~SQL,WITH dates AS (@@ -338,34 +354,19 @@ class AdminDashboardSiteTrafficAND date >= CAST(:start_date AS date)AND date <= CAST(:end_date AS date)GROUP BY date+ ),+ human_pageviews AS (+ #{human_pageviews_sql(range_start_date, range_end_date)})SELECTdates.date,GREATEST(0,- COALESCE(- SUM(- CASE- WHEN dates.date < :beacon_cutover_date AND ar.req_type = :logged_in_req_type THEN ar.count- WHEN dates.date >= :beacon_cutover_date AND ar.req_type = :logged_in_beacon_req_type THEN ar.count- ELSE 0- END- ),- 0- ) - COALESCE(MAX(lc.logged_in), 0)+ COALESCE(MAX(hp.logged_in), 0) - COALESCE(MAX(lc.logged_in), 0))::bigint AS logged_in,GREATEST(0,- COALESCE(- SUM(- CASE- WHEN dates.date < :beacon_cutover_date AND ar.req_type = :anonymous_req_type THEN ar.count- WHEN dates.date >= :beacon_cutover_date AND ar.req_type = :anonymous_beacon_req_type THEN ar.count- ELSE 0- END- ),- 0- ) - COALESCE(MAX(lc.anonymous), 0)+ COALESCE(MAX(hp.anonymous), 0) - COALESCE(MAX(lc.anonymous), 0))::bigint AS anonymous,COALESCE(SUM(CASE WHEN ar.req_type = :crawler_req_type THEN ar.count ELSE 0 END), 0)::bigint AS crawlers,COALESCE(SUM(CASE WHEN ar.req_type = :embedded_req_type THEN ar.count ELSE 0 END), 0)::bigint AS embedded,@@ -376,14 +377,8 @@ class AdminDashboardSiteTrafficFROM datesLEFT JOIN application_requests arON ar.date = dates.date- AND ar.req_type IN (- :logged_in_req_type,- :anonymous_req_type,- :logged_in_beacon_req_type,- :anonymous_beacon_req_type,- :crawler_req_type,- :embedded_req_type- )+ AND ar.req_type IN (:crawler_req_type, :embedded_req_type)+ LEFT JOIN human_pageviews hp ON hp.date = dates.dateLEFT JOIN likely_crawlers lc ON lc.date = dates.dateGROUP BY dates.dateORDER BY dates.date ASC@@ -392,11 +387,6 @@ class AdminDashboardSiteTrafficend_date: range_end_date,likely_crawlers_enabled: likely_crawlers_enabled?,login_required: login_required?,- beacon_cutover_date: cutover_date,- logged_in_req_type: selected_request_types[:logged_in],- anonymous_req_type: selected_request_types[:anonymous],- logged_in_beacon_req_type: beacon_request_types[:logged_in],- anonymous_beacon_req_type: beacon_request_types[:anonymous],crawler_req_type: selected_request_types[:crawlers],embedded_req_type: selected_request_types[:embedded],)diff --git a/spec/lib/statistics_spec.rb b/spec/lib/statistics_spec.rbindex aaea0ab4843..bc96a220631 100644--- a/spec/lib/statistics_spec.rb+++ b/spec/lib/statistics_spec.rb@@ -212,7 +212,7 @@ RSpec.describe Statistics doafter { ApplicationRequest.disable }- it "preserves piggyback visitor estimates on the first partial beacon day" do+ it "estimates visitors from beacon counters once beacons record traffic" doApplicationRequest.create!(date: date.to_date,req_type: :page_view_anon_browser_beacon,@@ -225,7 +225,7 @@ RSpec.describe Statistics do)freeze_time(date) do- expect(described_class.visitors).to eq(last_day: 3, "7_days": 6, "30_days": 9)+ expect(described_class.visitors).to eq(last_day: 5, "7_days": 8, "30_days": 11)endend@@ -301,7 +301,7 @@ RSpec.describe Statistics doafter { ApplicationRequest.disable }- it "preserves piggyback EU visitor estimates on the first partial beacon day" do+ it "estimates EU visitors from beacon counters once beacons record traffic" doApplicationRequest.create!(date: date.to_date,req_type: :page_view_anon_browser_beacon,@@ -314,7 +314,7 @@ RSpec.describe Statistics do)freeze_time(date) do- expect(described_class.eu_visitors).to eq(last_day: 3, "7_days": 3, "30_days": 5)+ expect(described_class.eu_visitors).to eq(last_day: 5, "7_days": 4, "30_days": 6)endenddiff --git a/spec/models/application_request_spec.rb b/spec/models/application_request_spec.rbindex 9c3bf1b02c7..89a7ec743ce 100644--- a/spec/models/application_request_spec.rb+++ b/spec/models/application_request_spec.rb@@ -17,29 +17,38 @@ RSpec.describe ApplicationRequest doenddescribe ".browser_pageviews" do- it "preserves the first partial beacon day and switches both cohorts the following day" do+ it "reads days with beacon traffic from beacons and the rest from piggyback counters" dotoday = Date.currenthistory =described_class.create!(date: today - 2, req_type: :page_view_anon_browser, count: 4)described_class.create!(date: today - 2, req_type: :page_view_anon_browser_beacon, count: 0)- first_day =- described_class.create!(date: today - 1, req_type: :page_view_anon_browser, count: 1000)- described_class.create!(date: today - 1, req_type: :page_view_anon_browser_beacon, count: 50)- described_class.create!(- date: today - 1,- req_type: :page_view_logged_in_browser_beacon,- count: 5,- )- beacon =- described_class.create!(date: today, req_type: :page_view_anon_browser_beacon, count: 8)- described_class.create!(date: today, req_type: :page_view_anon_browser, count: 3)+ anon_beacon =+ described_class.create!(+ date: today - 1,+ req_type: :page_view_anon_browser_beacon,+ count: 50,+ )+ logged_in_beacon =+ described_class.create!(+ date: today - 1,+ req_type: :page_view_logged_in_browser_beacon,+ count: 5,+ )+ described_class.create!(date: today - 1, req_type: :page_view_anon_browser, count: 1000)described_class.create!(date: today,req_type: :page_view_anon_browser_mobile_beacon,count: 2,)-- expect(described_class.browser_pageviews).to contain_exactly(history, first_day, beacon)+ mobile_only_day =+ described_class.create!(date: today, req_type: :page_view_anon_browser, count: 3)++ expect(described_class.browser_pageviews).to contain_exactly(+ history,+ anon_beacon,+ logged_in_beacon,+ mobile_only_day,+ )endit "includes the first beacon day when piggyback counters are absent or zero" do@@ -51,6 +60,25 @@ RSpec.describe ApplicationRequest doexpect(described_class.browser_pageviews).to contain_exactly(beacon)end+ it "keeps using piggyback counters on days a lone early beacon did not cover" do+ today = Date.current+ described_class.create!(date: today - 30, req_type: :page_view_anon_browser_beacon, count: 1)+ history = described_class.create!(date: today, req_type: :page_view_anon_browser, count: 100)++ expect(described_class.browser_pageviews.where(date: today)).to contain_exactly(history)+ end++ it "decides anonymous and logged in traffic separately on the same day" do+ today = Date.current+ logged_in =+ described_class.create!(date: today, req_type: :page_view_logged_in_browser, count: 40)+ described_class.create!(date: today, req_type: :page_view_anon_browser, count: 60)+ anon_beacon =+ described_class.create!(date: today, req_type: :page_view_anon_browser_beacon, count: 25)++ expect(described_class.browser_pageviews).to contain_exactly(logged_in, anon_beacon)+ end+it "uses piggyback history when no beacons have been recorded" dohistory =described_class.create!(date: Date.current, req_type: :page_view_anon_browser, count: 4)diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rbindex 788db5c9c23..8714dc634d6 100644--- a/spec/models/report_spec.rb+++ b/spec/models/report_spec.rb@@ -406,7 +406,7 @@ RSpec.describe Report dolet(:report) { Report.find("page_view_total_reqs") }- it "combines historical piggyback totals with beacon totals without counting overlap" do+ it "reads days with beacon traffic from beacons and the rest from piggyback counters" doSiteSetting.use_legacy_pageviews = falseApplicationRequest.create!(date: Date.current - 2,@@ -420,8 +420,8 @@ RSpec.describe Report docount: 8,)- expect(report.data).to eq([{ x: Date.current - 2, y: 4 }, { x: Date.current, y: 3 }])- expect(report.total).to eq(7)+ expect(report.data).to eq([{ x: Date.current - 2, y: 4 }, { x: Date.current, y: 8 }])+ expect(report.total).to eq(12)endcontext "with no data" do@@ -1830,31 +1830,51 @@ RSpec.describe Report doexpect(series["page_view_other"]).to eq([{ x: Date.current, y: 2 }])end- it "uses historical piggyback counts before switching both browser series to beacons" do+ it "keeps piggyback counts in the browser series after an isolated day of beacon data" do+ ApplicationRequest.create!(+ date: Date.current - 2,+ req_type: :page_view_anon_browser_beacon,+ count: 1,+ )+ ApplicationRequest.create!(date: Date.current, req_type: :page_view_anon_browser, count: 30)+ ApplicationRequest.create!(date: Date.current, req_type: :page_view_anon, count: 30)++ series = reports.data.to_h { |entry| [entry[:req], entry[:data]] }++ expect(series["page_view_anon_browser"].last).to eq({ x: Date.current, y: 30 })+ expect(series["page_view_other"].last).to eq({ x: Date.current, y: 0 })+ end++ it "reads days with beacon traffic from beacons and the rest from piggyback counters" doApplicationRequest.create!(date: Date.current - 2,req_type: :page_view_anon_browser,count: 4,)- ApplicationRequest.create!(date: Date.current, req_type: :page_view_anon_browser, count: 3)ApplicationRequest.create!(- date: Date.current,+ date: Date.current - 2,req_type: :page_view_logged_in_browser,count: 6,)+ ApplicationRequest.create!(date: Date.current, req_type: :page_view_anon_browser, count: 3)ApplicationRequest.create!(date: Date.current,req_type: :page_view_anon_browser_beacon,count: 8,)+ ApplicationRequest.create!(+ date: Date.current,+ req_type: :page_view_logged_in_browser_beacon,+ count: 2,+ )series = reports.data.to_h { |entry| [entry[:req], entry[:data]] }expect(series["page_view_anon_browser"]).to eq(- [{ x: Date.current - 2, y: 4 }, { x: Date.current, y: 3 }],+ [{ x: Date.current - 2, y: 4 }, { x: Date.current, y: 8 }],)expect(series["page_view_logged_in_browser"]).to eq(- [{ x: Date.current - 2, y: 0 }, { x: Date.current, y: 6 }],+ [{ x: Date.current - 2, y: 6 }, { x: Date.current, y: 2 }],)expect(series["page_view_other"]).to eq([{ x: Date.current - 2, y: 0 }, { x: Date.current, y: 0 }],diff --git a/spec/services/admin_dashboard_site_traffic_spec.rb b/spec/services/admin_dashboard_site_traffic_spec.rbindex 25012786290..bf029b698be 100644--- a/spec/services/admin_dashboard_site_traffic_spec.rb+++ b/spec/services/admin_dashboard_site_traffic_spec.rb@@ -189,20 +189,36 @@ RSpec.describe AdminDashboardSiteTraffic do)end- it "switches to beacon counters after the first day with beacon data" do+ it "reads days with beacon traffic from beacons and the rest from piggyback counters" doFabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 20)Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-01", count: 15)+ Fabricate(:logged_in_browser_application_request, date: "2026-05-02", count: 25)+ Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-02", count: 30)+ Fabricate(:logged_in_browser_application_request, date: "2026-05-03", count: 40)++ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")++ expect(traffic_series_data(response, :logged_in)).to eq(+ [+ traffic_point("2026-05-01", 15),+ traffic_point("2026-05-02", 30),+ traffic_point("2026-05-03", 40),+ ],+ )+ end++ it "keeps counting piggyback traffic after an isolated day of beacon data" do+ Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-01", count: 1)Fabricate(:logged_in_browser_application_request, date: "2026-05-02", count: 30)- Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-02", count: 25)Fabricate(:logged_in_browser_application_request, date: "2026-05-03", count: 40)response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")expect(traffic_series_data(response, :logged_in)).to eq([- traffic_point("2026-05-01", 20),- traffic_point("2026-05-02", 25),- traffic_point("2026-05-03", 0),+ traffic_point("2026-05-01", 1),+ traffic_point("2026-05-02", 30),+ traffic_point("2026-05-03", 40),],)end