A platform for community discussion. Free, open, simple.
DEV: Consolidate `BrowserPageviewEvent` persistence on beacons (#43244)
Krzysztof Kotlarekddfc1d99e74d
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rbindex fec405e2350..dd6c362fac1 100644--- a/app/helpers/application_helper.rb+++ b/app/helpers/application_helper.rb@@ -427,9 +427,6 @@ module ApplicationHelperenddef discourse_pageview_tracking_meta_tags- if !SiteSetting.trigger_browser_pageview_events && !SiteSetting.persist_browser_pageview_events- return ""- endreturn "" if Rails.env.development? && ENV["TRACK_REQUESTS"].blank?tags = +""@@ -437,13 +434,6 @@ module ApplicationHelpername: "discourse-track-view-session-id",content: track_view_session_id_placeholder,)- if UpcomingChanges.enabled?(:dashboard_improvements)- tags << tag.meta(name: "discourse-beacon-pageview-enabled", content: "true")- end-- if SiteSetting.persist_browser_pageview_events- tags << tag.meta(name: "discourse-engagement-tracking-enabled", content: "true")- endtags.html_safeenddiff --git a/app/jobs/scheduled/flush_browser_pageview_events.rb b/app/jobs/scheduled/flush_browser_pageview_events.rbindex 80ac34ea68b..5472e47dcb2 100644--- a/app/jobs/scheduled/flush_browser_pageview_events.rb+++ b/app/jobs/scheduled/flush_browser_pageview_events.rb@@ -7,7 +7,6 @@ module JobsMAX_FLUSH_SECONDS = 50def execute(args)- return if !SiteSetting.persist_browser_pageview_eventsreturn if Discourse.pg_readonly_mode?deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + MAX_FLUSH_SECONDSdiff --git a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb b/app/jobs/scheduled/maintain_browser_pageview_rollups.rbindex 2992e2112bf..539357f70d1 100644--- a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb+++ b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb@@ -7,8 +7,6 @@ module Jobscluster_concurrency 1def execute(_args)- return if !SiteSetting.persist_browser_pageview_events-aggregate_pageviewsaggregate_engagementaggregate_crawlers@@ -97,10 +95,7 @@ module Jobsenddef earliest_event_date- BrowserPageviewEvent- .where(BrowserPageviewEvent.rollup_source_condition)- .minimum(:created_at)- &.to_date+ BrowserPageviewEvent.minimum(:created_at)&.to_dateenddef backfill_referrers(recompute_entry_urls:)@@ -133,7 +128,6 @@ module JobsSELECT id, referrerFROM browser_pageview_eventsWHERE referrer IS NOT NULL- AND #{BrowserPageviewEvent.rollup_source_condition}AND (normalized_referrer_version IS NULLOR normalized_referrer_version < :version@@ -186,7 +180,6 @@ module JobsFROM browser_pageview_events eWHERE e.created_at >= touched_dates.dateAND e.created_at < touched_dates.date + 1- AND #{BrowserPageviewEvent.rollup_source_condition(table: "e")}AND e.referrer IS NOT NULLAND NOT EXISTS (SELECT 1@@ -260,7 +253,6 @@ module JobsFROM browser_pageview_events eventsWHERE events.created_at >= touched_dates.dateAND events.created_at < touched_dates.date + 1- AND #{BrowserPageviewEvent.rollup_source_condition(table: "events")}AND (((@@ -323,7 +315,6 @@ module JobsSELECT id, user_agentFROM browser_pageview_eventsWHERE created_at >= :retention_cutoff- AND #{BrowserPageviewEvent.rollup_source_condition}AND browser IS NULLORDER BY created_at DESC, id DESCLIMIT :limit@@ -336,7 +327,6 @@ module JobsSELECT id, urlFROM browser_pageview_eventsWHERE created_at >= :retention_cutoff- AND #{BrowserPageviewEvent.rollup_source_condition}AND (normalized_url_version IS NULLOR normalized_url_version < :versiondiff --git a/app/models/application_request.rb b/app/models/application_request.rbindex 962ad285769..f3ff8eb0f84 100644--- a/app/models/application_request.rb+++ b/app/models/application_request.rb@@ -28,6 +28,45 @@ class ApplicationRequest < ActiveRecord::Baseinclude CachedCounting+ def self.browser_pageviews+ legacy_types = %i[page_view_anon_browser page_view_logged_in_browser]+ beacon_types = %i[page_view_anon_browser_beacon page_view_logged_in_browser_beacon]+ cutover_date = browser_pageview_beacon_cutover_date++ return where(req_type: legacy_types) if cutover_date.nil?++ # Both transports can record the same visit, so select one source for each date.+ where(req_type: legacy_types).where("date < ?", cutover_date).or(+ where(req_type: beacon_types).where("date >= ?", cutover_date),+ )+ end++ def self.browser_pageview_beacon_cutover_date+ first_beacon_date =+ where(req_type: %i[page_view_anon_browser_beacon page_view_logged_in_browser_beacon]).where(+ "count > 0",+ ).minimum(:date)+ return if first_beacon_date.nil?++ # The first beacon day may be partial. Prefer existing piggyback traffic for that+ # whole day, but retain the first day on sites that only recorded beacons.+ if where(+ date: first_beacon_date,+ req_type: %i[page_view_anon_browser page_view_logged_in_browser],+ ).where("count > 0").exists?+ first_beacon_date.next_day+ else+ first_beacon_date+ end+ end++ def self.browser_pageview_count_for_period(type, since)+ browser_pageviews+ .where(req_type: [type, "#{type}_beacon"])+ .where("date >= ?", since)+ .sum(:count)+ end+def self.disable@disabled = trueenddiff --git a/app/models/browser_pageview_country_daily_rollup.rb b/app/models/browser_pageview_country_daily_rollup.rbindex 7743fc50dd6..606029c6b9f 100644--- a/app/models/browser_pageview_country_daily_rollup.rb+++ b/app/models/browser_pageview_country_daily_rollup.rb@@ -21,7 +21,6 @@ class BrowserPageviewCountryDailyRollup < ActiveRecord::BaseFROM browser_pageview_eventsWHERE created_at >= :start_dateAND created_at < :end_date- AND #{BrowserPageviewEvent.rollup_source_condition}GROUP BY date, country_codeON CONFLICT (date, country_code) DO UPDATESET count = EXCLUDED.count,diff --git a/app/models/browser_pageview_crawler_daily_rollup.rb b/app/models/browser_pageview_crawler_daily_rollup.rbindex 127268edc00..bba53fd997a 100644--- a/app/models/browser_pageview_crawler_daily_rollup.rb+++ b/app/models/browser_pageview_crawler_daily_rollup.rb@@ -15,7 +15,6 @@ class BrowserPageviewCrawlerDailyRollup < ActiveRecord::BaseWHERE created_at >= :start_dateAND created_at < :end_dateAND score > :threshold- AND #{BrowserPageviewEvent.rollup_source_condition}GROUP BY date, logged_inON CONFLICT (date, logged_in) DO UPDATESET count = EXCLUDED.countdiff --git a/app/models/browser_pageview_entry_url_daily_rollup.rb b/app/models/browser_pageview_entry_url_daily_rollup.rbindex 0f36bcb5ee0..00a27c95780 100644--- a/app/models/browser_pageview_entry_url_daily_rollup.rb+++ b/app/models/browser_pageview_entry_url_daily_rollup.rb@@ -8,12 +8,6 @@ class BrowserPageviewEntryUrlDailyRollup < ActiveRecord::Basereturn if start_date > end_dateexclusive_end_date = end_date + 1- source_condition =- BrowserPageviewEvent.rollup_source_condition(- table: "events",- start_date:,- end_date: exclusive_end_date,- )transaction doDB.exec(<<~SQL, start_date:, end_date: exclusive_end_date)DELETE FROM browser_pageview_entry_url_daily_rollups rollup@@ -24,7 +18,6 @@ class BrowserPageviewEntryUrlDailyRollup < ActiveRecord::BaseFROM browser_pageview_events eventsWHERE events.created_at >= rollup.dateAND events.created_at < rollup.date + 1- AND #{source_condition})SQL@@ -52,7 +45,6 @@ class BrowserPageviewEntryUrlDailyRollup < ActiveRecord::BaseFROM browser_pageview_events eventsWHERE events.created_at >= :start_dateAND events.created_at < :end_date- AND #{source_condition}AND events.normalized_url IS NOT NULLAND (NULLIF(events.referrer, '') IS NULL@@ -75,7 +67,6 @@ class BrowserPageviewEntryUrlDailyRollup < ActiveRecord::Baseearliest_source_date =BrowserPageviewEvent.where("created_at >= ?", BrowserPageviewEvent.retention_cutoff)- .where(BrowserPageviewEvent.rollup_source_condition).minimum(:created_at)&.to_datereturn start_date if earliest_source_date.nil? || start_date > earliest_source_datediff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rbindex 377291b5f13..8e8e3dfd6f8 100644--- a/app/models/browser_pageview_event.rb+++ b/app/models/browser_pageview_event.rb@@ -1,6 +1,8 @@# frozen_string_literal: trueclass BrowserPageviewEvent < ActiveRecord::Base+ self.ignored_columns += %i[source] # TODO(02-2027): Remove after the column is dropped+MAX_SESSION_ID_LENGTH = 32MAX_URL_LENGTH = 2000MAX_REFERRER_LENGTH = 2000@@ -10,8 +12,6 @@ class BrowserPageviewEvent < ActiveRecord::BaseMAX_NORMALIZED_REFERRER_LENGTH = 2000MAX_NORMALIZED_URL_LENGTH = 2000RETENTION_PERIOD = 3.months- SOURCE_PIGGYBACK = 1- SOURCE_BEACON = 2BROWSER_UNKNOWN = 0BROWSERS = {unknown: BROWSER_UNKNOWN,@@ -34,7 +34,6 @@ class BrowserPageviewEvent < ActiveRecord::BaseREDIS_QUEUE_MAX_SIZE = 1_000_000REDIS_QUEUE_TTL = 1.day- enum :source, { piggyback: SOURCE_PIGGYBACK, beacon: SOURCE_BEACON }, scopes: falseenum :browser, BROWSERS, prefix: true, scopes: falseclass << self@@ -64,7 +63,9 @@ class BrowserPageviewEvent < ActiveRecord::Basequeued_attributes = []entries.each do |entry|- queued_attributes << attributes_from_payload(deserialize_payload(entry))+ payload = deserialize_payload(entry)+ # Old processes can leave piggyback events in Redis during deployment.+ queued_attributes << (payload[:source] == 1 ? nil : attributes_from_payload(payload))processed += 1rescue => eRails.logger.error("Discarding queued BrowserPageviewEvent: #{e.message}")@@ -97,29 +98,6 @@ class BrowserPageviewEvent < ActiveRecord::BaseDiscourse.redis.llen(REDIS_QUEUE_KEY).to_iend- def beacon_cutover_date- return if SiteSetting.use_legacy_pageviews- return if !UpcomingChanges.enabled?(:dashboard_improvements)- if !SiteSetting.trigger_browser_pageview_events &&- !SiteSetting.persist_browser_pageview_events- return- end-- enabled_at = [- UpcomingChangeEvent.where(- upcoming_change_name: "dashboard_improvements",- event_type: %i[manual_opt_in automatically_promoted],- ).maximum(:created_at),- SiteSetting.where(name: "dashboard_improvements").maximum(:updated_at),- ].compact.max-- return enabled_at.utc.to_date.tomorrow if enabled_at-- ApplicationRequest.where(- req_type: %w[page_view_logged_in_browser_beacon page_view_anon_browser_beacon],- ).minimum(:date)- end-def clear_queued!Discourse.redis.del(REDIS_QUEUE_KEY)end@@ -179,7 +157,6 @@ class BrowserPageviewEvent < ActiveRecord::Baselanguage: payload[:language]&.slice(0, MAX_LANGUAGE_LENGTH),session_id: payload[:session_id]&.slice(0, MAX_SESSION_ID_LENGTH),topic_id: payload[:topic_id],- source: payload[:source],occurred_at: payload[:occurred_at],}end@@ -211,7 +188,6 @@ class BrowserPageviewEvent < ActiveRecord::Basesession_id: payload[:session_id]&.slice(0, MAX_SESSION_ID_LENGTH),user_id: payload[:user_id],topic_id: payload[:topic_id],- source: payload[:source],created_at: payload[:occurred_at],}end@@ -241,23 +217,6 @@ class BrowserPageviewEvent < ActiveRecord::BaseRETENTION_PERIOD.ago.beginning_of_dayend- def self.rollup_source_condition(table: nil, start_date: nil, end_date: nil)- prefix = table ? "#{table}." : ""- cutover_date = beacon_cutover_date- return "#{prefix}source = #{SOURCE_PIGGYBACK}" if cutover_date.nil?- return "#{prefix}source = #{SOURCE_PIGGYBACK}" if end_date && end_date.to_date <= cutover_date- return "#{prefix}source = #{SOURCE_BEACON}" if start_date && start_date.to_date >= cutover_date-- sanitize_sql_array(- [- "(#{prefix}created_at < ? AND #{prefix}source = #{SOURCE_PIGGYBACK} " \- "OR #{prefix}created_at >= ? AND #{prefix}source = #{SOURCE_BEACON})",- cutover_date,- cutover_date,- ],- )- end-def self.rollup_count_sqllogged_in_only = SiteSetting.login_requiredcount_column = logged_in_only ? "logged_in_count" : "count"@@ -306,7 +265,6 @@ end# normalized_url_version :integer# referrer :string(2000)# score :integer-# source :integer default("piggyback"), not null# url :string(2000) not null# user_agent :string(1000) not null# created_at :datetime not null@@ -317,8 +275,9 @@ end# Indexes## idx_bpe_beacon_created_at_id (created_at,id) WHERE (source = 2)-# idx_bpe_browser_backfill (source,created_at DESC,id DESC) WHERE (browser IS NULL)+# idx_bpe_browser_backfill (created_at,id) WHERE (browser IS NULL)# idx_bpe_created_at_country_code (created_at,country_code)+# idx_bpe_created_at_id (created_at,id)# idx_bpe_created_at_normalized_referrer (created_at,normalized_referrer)# idx_bpe_created_at_session_id (created_at,session_id,source)# idx_bpe_ip_ua_created_at (ip_address,user_agent,created_at)diff --git a/app/models/browser_pageview_referrer_daily_rollup.rb b/app/models/browser_pageview_referrer_daily_rollup.rbindex a72b6df10c6..4e884736f44 100644--- a/app/models/browser_pageview_referrer_daily_rollup.rb+++ b/app/models/browser_pageview_referrer_daily_rollup.rb@@ -21,7 +21,6 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::BaseFROM browser_pageview_eventsWHERE created_at >= :start_dateAND created_at < :end_date- AND #{BrowserPageviewEvent.rollup_source_condition}GROUP BY date, normalized_referrerON CONFLICT (date, normalized_referrer) DO UPDATESET count = EXCLUDED.count,@@ -47,7 +46,6 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::BaseFROM browser_pageview_events eWHERE e.created_at >= d.dateAND e.created_at < d.date + 1- AND #{BrowserPageviewEvent.rollup_source_condition(table: "e")})SQLreturn if dates.empty?diff --git a/app/models/browser_pageview_session_engagement_daily_rollup.rb b/app/models/browser_pageview_session_engagement_daily_rollup.rbindex cd2df7fe92a..9c9c69b14d5 100644--- a/app/models/browser_pageview_session_engagement_daily_rollup.rb+++ b/app/models/browser_pageview_session_engagement_daily_rollup.rb@@ -11,8 +11,6 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Basedef self.aggregate(start_date:, end_date:)start_date = start_date.to_dateend_date = end_date.to_date + 1- source_condition = BrowserPageviewEvent.rollup_source_condition-transaction doDB.exec(<<~SQL, start_date:, end_date:)DELETE FROM browser_pageview_session_engagement_daily_rollups rollup@@ -23,7 +21,6 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::BaseFROM browser_pageview_eventsWHERE created_at >= rollup.dateAND created_at < rollup.date + 1- AND #{source_condition})SQL@@ -34,7 +31,6 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::BaseFROM browser_pageview_eventsWHERE created_at >= :start_dateAND created_at < LEAST(:end_date::timestamp, :session_started_before::timestamp)- AND #{source_condition}),session_pageviews AS (SELECT sp.*@@ -48,7 +44,6 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Basebool_or(#{CrawlerScorer.likely_crawler_condition(table: "bpe")}) AS likely_crawlerFROM browser_pageview_events bpeWHERE bpe.session_id = active_sessions.session_id- AND #{BrowserPageviewEvent.rollup_source_condition(table: "bpe")}GROUP BY bpe.session_idHAVING MIN(bpe.created_at) >= :start_date) spdiff --git a/app/models/category_activity_daily_rollup.rb b/app/models/category_activity_daily_rollup.rbindex 82536ae1fbe..736d8433555 100644--- a/app/models/category_activity_daily_rollup.rb+++ b/app/models/category_activity_daily_rollup.rb@@ -126,7 +126,6 @@ class CategoryActivityDailyRollup < ActiveRecord::BaseWHERE bpe.created_at >= :start_dateAND bpe.created_at < :end_dateAND #{CrawlerScorer.likely_crawler_condition(table: "bpe")}- AND #{BrowserPageviewEvent.rollup_source_condition(table: "bpe")}AND #{ELIGIBLE_TOPICS}GROUP BY 1, 2)@@ -188,7 +187,6 @@ class CategoryActivityDailyRollup < ActiveRecord::BaseFROM browser_pageview_eventsWHERE created_at >= :start_dateAND created_at < :end_date- AND #{BrowserPageviewEvent.rollup_source_condition}SQLscope = where(date: start_date..end_date).where("likely_crawler_page_views > 0")diff --git a/app/models/concerns/reports/site_traffic.rb b/app/models/concerns/reports/site_traffic.rbindex f6c39483e70..70075f2f75b 100644--- a/app/models/concerns/reports/site_traffic.rb+++ b/app/models/concerns/reports/site_traffic.rb@@ -16,22 +16,21 @@ module Reports::SiteTrafficdef report_site_traffic(report)report.modes = [Report::MODES[:stacked_chart]]- first_browser_pageview_date =- DB.query_single(- <<~SQL,- SELECT date FROM application_requests- WHERE req_type = :page_view_logged_in_browser OR req_type = :page_view_anon_browser ORDER BY date LIMIT 1- SQL- page_view_logged_in_browser: ApplicationRequest.req_types[:page_view_logged_in_browser],- page_view_anon_browser: ApplicationRequest.req_types[:page_view_anon_browser],- ).first+ browser_pageviews = ApplicationRequest.browser_pageviews+ first_browser_pageview_date = browser_pageviews.minimum(:date)likely_crawlers_enabled = CrawlerScorer.enabled?data =DB.query(<<~SQL,- WITH likely_crawlers AS (+ WITH browser_pageviews AS (+ SELECT date,+ SUM(count) FILTER (WHERE req_type IN (:logged_in_browser_types)) AS logged_in,+ SUM(count) FILTER (WHERE req_type IN (:anonymous_browser_types)) AS anonymous+ FROM (#{browser_pageviews.where(date: report.start_date..report.end_date).to_sql}) requests+ GROUP BY date+ ), likely_crawlers AS (SELECTdate,COALESCE(SUM(count) FILTER (WHERE logged_in), 0)::bigint AS logged_in,@@ -46,12 +45,12 @@ module Reports::SiteTrafficar.date,GREATEST(0,- SUM(CASE WHEN ar.req_type = :page_view_logged_in_browser THEN ar.count ELSE 0 END)+ COALESCE(MAX(bp.logged_in), 0)- COALESCE(MAX(lc.logged_in), 0)) AS page_view_logged_in_browser,GREATEST(0,- SUM(CASE WHEN ar.req_type = :page_view_anon_browser THEN ar.count ELSE 0 END)+ COALESCE(MAX(bp.anonymous), 0)- COALESCE(MAX(lc.anonymous), 0)) AS page_view_anon_browser,(COALESCE(MAX(lc.logged_in), 0) + COALESCE(MAX(lc.anonymous), 0)) AS page_view_likely_crawler,@@ -62,13 +61,12 @@ module Reports::SiteTrafficSUM(CASE WHEN ar.req_type = :page_view_anon THEN ar.countWHEN ar.req_type = :page_view_logged_in THEN ar.count- WHEN ar.req_type = :page_view_anon_browser THEN -ar.count- WHEN ar.req_type = :page_view_logged_in_browser THEN -ar.countELSE 0END- )+ ) - COALESCE(MAX(bp.logged_in), 0) - COALESCE(MAX(bp.anonymous), 0)) AS page_view_otherFROM application_requests ar+ LEFT JOIN browser_pageviews bp ON bp.date = ar.dateLEFT JOIN likely_crawlers lc ON lc.date = ar.dateWHERE ar.date >= :start_date AND ar.date <= :end_date AND ar.date >= :first_browser_pageview_date@@ -81,8 +79,16 @@ module Reports::SiteTrafficpage_view_anon: ApplicationRequest.req_types[:page_view_anon],page_view_crawler: ApplicationRequest.req_types[:page_view_crawler],page_view_logged_in: ApplicationRequest.req_types[:page_view_logged_in],- page_view_anon_browser: ApplicationRequest.req_types[:page_view_anon_browser],- page_view_logged_in_browser: ApplicationRequest.req_types[:page_view_logged_in_browser],+ anonymous_browser_types:+ ApplicationRequest.req_types.values_at(+ "page_view_anon_browser",+ "page_view_anon_browser_beacon",+ ),+ logged_in_browser_types:+ ApplicationRequest.req_types.values_at(+ "page_view_logged_in_browser",+ "page_view_logged_in_browser_beacon",+ ),page_view_embed: ApplicationRequest.req_types[:page_view_embed],first_browser_pageview_date: first_browser_pageview_date,)diff --git a/app/models/report.rb b/app/models/report.rbindex ce6ba8a90fe..3bcf996ba81 100644--- a/app/models/report.rb+++ b/app/models/report.rb@@ -48,17 +48,9 @@ class Report]ADMIN_ONLY_RELATED_ITEMS_REPORTS = %w[new_contributors signups]IP_ADDRESS_REPORTS = %w[suspicious_logins]- BROWSER_PAGEVIEW_REPORTS = %w[- top_countries_by_browser_pageviews- top_entry_urls- top_referrers_by_browser_pageviews- ]def self.hidden?(type, guardian:)return true if !guardian.is_admin? && ADMIN_ONLY_REPORTS.include?(type)- if BROWSER_PAGEVIEW_REPORTS.include?(type) && !SiteSetting.persist_browser_pageview_events- return true- endhidden_reports =SiteSetting.use_legacy_pageviews ? HIDDEN_PAGEVIEW_REPORTS : HIDDEN_LEGACY_PAGEVIEW_REPORTS@@ -503,12 +495,7 @@ class Report# too. This is to reflect what is shown in the "Site traffic" report# by default.def self.page_view_requests- ApplicationRequest.where(- req_type: [- ApplicationRequest.req_types[:page_view_anon_browser],- ApplicationRequest.req_types[:page_view_logged_in_browser],- ].flatten,- )+ ApplicationRequest.browser_pageviewsenddef self.report_about(report, subject_class, report_method = :count_per_day)diff --git a/app/services/admin_dashboard_site_traffic.rb b/app/services/admin_dashboard_site_traffic.rbindex 7ccdf82128e..39a740d29e8 100644--- a/app/services/admin_dashboard_site_traffic.rb+++ b/app/services/admin_dashboard_site_traffic.rb@@ -40,7 +40,7 @@ class AdminDashboardSiteTrafficpageview_series: pageview_series(current_rows, include_embedded: include_embedded),}- if SiteSetting.persist_browser_pageview_events && !SiteSetting.use_legacy_pageviews+ if !SiteSetting.use_legacy_pageviewstop_countries = fetch_card("top_countries_by_browser_pageviews")response[:top_countries] = top_countries if top_countries@@ -120,10 +120,8 @@ class AdminDashboardSiteTrafficdirect_traffic = direct_traffic_valuekpis[:direct_traffic] = { value: direct_traffic } if !direct_traffic.nil?- if SiteSetting.persist_browser_pageview_events- kpis[:bounce_rate] = { value: bounce_rate_value }- kpis[:average_session_duration_seconds] = { value: average_session_duration_value }- end+ kpis[:bounce_rate] = { value: bounce_rate_value }+ kpis[:average_session_duration_seconds] = { value: average_session_duration_value }kpisend@@ -183,8 +181,6 @@ class AdminDashboardSiteTrafficenddef direct_traffic_value- return nil if !SiteSetting.persist_browser_pageview_events-count_sql = BrowserPageviewEvent.rollup_count_sqlrow = DB.query(<<~SQL, start_date: start_date.to_date, end_date: end_date.to_date).first@@ -312,9 +308,10 @@ class AdminDashboardSiteTrafficenddef beacon_cutover_date+ return if SiteSetting.use_legacy_pageviewsreturn @beacon_cutover_date if defined?(@beacon_cutover_date)- @beacon_cutover_date = BrowserPageviewEvent.beacon_cutover_date+ @beacon_cutover_date = ApplicationRequest.browser_pageview_beacon_cutover_dateenddef traffic_rows(range_start_date, range_end_date)diff --git a/app/services/admin_dashboard_site_traffic_explorer.rb b/app/services/admin_dashboard_site_traffic_explorer.rbindex afc9f2fc003..6911f0ebfbb 100644--- a/app/services/admin_dashboard_site_traffic_explorer.rb+++ b/app/services/admin_dashboard_site_traffic_explorer.rb@@ -219,9 +219,6 @@ class AdminDashboardSiteTrafficExplorerenddef query(start_date:, end_date:)- source_condition =- BrowserPageviewEvent.rollup_source_condition(table: "bpe", start_date:, end_date:)-<<~SQLWITH population AS MATERIALIZED (SELECT@@ -240,7 +237,6 @@ class AdminDashboardSiteTrafficExplorerFROM browser_pageview_events bpeWHERE bpe.created_at >= :start_dateAND bpe.created_at < :end_date- AND #{source_condition}ORDER BY bpe.created_at DESC, bpe.id DESCLIMIT :cap),@@ -360,7 +356,6 @@ class AdminDashboardSiteTrafficExplorerFROM browser_pageview_events bpeWHERE bpe.created_at >= :start_dateAND bpe.created_at < :end_date- AND #{source_condition}AND (bpe.created_at, bpe.id) < (population_boundary.oldest_created_at,population_boundary.oldest_iddiff --git a/config/initializers/004-message_bus.rb b/config/initializers/004-message_bus.rbindex 803bd580a65..9fae4af3c46 100644--- a/config/initializers/004-message_bus.rb+++ b/config/initializers/004-message_bus.rb@@ -37,7 +37,7 @@ def setup_message_bus_env(env)"Access-Control-Allow-Origin" => cors_origin,"Access-Control-Allow-Methods" => "GET, POST","Access-Control-Allow-Headers" =>- "X-SILENCE-LOGGER, X-Shared-Session-Key, Dont-Chunk, Discourse-Present, Discourse-Deferred-Track-View",+ "X-SILENCE-LOGGER, X-Shared-Session-Key, Dont-Chunk, Discourse-Present","Access-Control-Max-Age" => "7200",}diff --git a/config/site_settings.yml b/config/site_settings.ymlindex f84f908d720..2af6ba4125b 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -607,12 +607,6 @@ basic:default: falsehidden: trueclient: true- trigger_browser_pageview_events:- default: false- hidden: true- persist_browser_pageview_events:- default: true- hidden: trueclean_up_browser_pageview_events:default: truehidden: truediff --git a/db/migrate/20260831011836_add_general_traffic_explorer_index_to_browser_pageview_events.rb b/db/migrate/20260831011836_add_general_traffic_explorer_index_to_browser_pageview_events.rbnew file mode 100644index 00000000000..c7d8a56fed3--- /dev/null+++ b/db/migrate/20260831011836_add_general_traffic_explorer_index_to_browser_pageview_events.rb@@ -0,0 +1,29 @@+# frozen_string_literal: true++class AddGeneralTrafficExplorerIndexToBrowserPageviewEvents < ActiveRecord::Migration[8.0]+ disable_ddl_transaction!++ INDEX_NAME = "idx_bpe_created_at_id"++ def up+ remove_index :browser_pageview_events,+ name: INDEX_NAME,+ algorithm: :concurrently,+ if_exists: true+ add_index :browser_pageview_events,+ %i[created_at id],+ name: INDEX_NAME,+ order: {+ created_at: :desc,+ id: :desc,+ },+ algorithm: :concurrently+ end++ def down+ remove_index :browser_pageview_events,+ name: INDEX_NAME,+ algorithm: :concurrently,+ if_exists: true+ end+enddiff --git a/db/migrate/20260831011839_replace_browser_backfill_index_on_browser_pageview_events.rb b/db/migrate/20260831011839_replace_browser_backfill_index_on_browser_pageview_events.rbnew file mode 100644index 00000000000..4cc8f9d3f97--- /dev/null+++ b/db/migrate/20260831011839_replace_browser_backfill_index_on_browser_pageview_events.rb@@ -0,0 +1,30 @@+# frozen_string_literal: true++class ReplaceBrowserBackfillIndexOnBrowserPageviewEvents < ActiveRecord::Migration[8.0]+ disable_ddl_transaction!++ INDEX_NAME = "idx_bpe_browser_backfill"++ def up+ remove_index :browser_pageview_events,+ name: INDEX_NAME,+ algorithm: :concurrently,+ if_exists: true+ add_index :browser_pageview_events,+ %i[created_at id],+ name: INDEX_NAME,+ order: {+ created_at: :desc,+ id: :desc,+ },+ where: "browser IS NULL",+ algorithm: :concurrently+ end++ def down+ remove_index :browser_pageview_events,+ name: INDEX_NAME,+ algorithm: :concurrently,+ if_exists: true+ end+enddiff --git a/db/migrate/20260831011840_set_browser_pageview_event_source_default_to_beacon.rb b/db/migrate/20260831011840_set_browser_pageview_event_source_default_to_beacon.rbnew file mode 100644index 00000000000..192e4bd7a0d--- /dev/null+++ b/db/migrate/20260831011840_set_browser_pageview_event_source_default_to_beacon.rb@@ -0,0 +1,35 @@+# frozen_string_literal: true++class SetBrowserPageviewEventSourceDefaultToBeacon < ActiveRecord::Migration[8.0]+ def up+ change_column_default :browser_pageview_events, :source, from: 1, to: 2++ execute <<~SQL+ CREATE SCHEMA IF NOT EXISTS discourse_functions;++ CREATE FUNCTION discourse_functions.skip_piggyback_browser_pageview_events()+ RETURNS trigger AS $$+ BEGIN+ IF NEW.source = 1 THEN+ RETURN NULL;+ END IF;++ RETURN NEW;+ END;+ $$ LANGUAGE plpgsql;++ CREATE TRIGGER skip_piggyback_browser_pageview_events+ BEFORE INSERT OR UPDATE OF source ON browser_pageview_events+ FOR EACH ROW WHEN (NEW.source IN (1, 2))+ EXECUTE FUNCTION discourse_functions.skip_piggyback_browser_pageview_events();+ SQL+ end++ def down+ execute <<~SQL+ DROP TRIGGER skip_piggyback_browser_pageview_events ON browser_pageview_events;+ DROP FUNCTION discourse_functions.skip_piggyback_browser_pageview_events();+ SQL+ change_column_default :browser_pageview_events, :source, from: 2, to: 1+ end+enddiff --git a/db/migrate/20260902075239_remove_browser_pageview_event_site_settings.rb b/db/migrate/20260902075239_remove_browser_pageview_event_site_settings.rbnew file mode 100644index 00000000000..b5ae3d3f9d2--- /dev/null+++ b/db/migrate/20260902075239_remove_browser_pageview_event_site_settings.rb@@ -0,0 +1,14 @@+# frozen_string_literal: true++class RemoveBrowserPageviewEventSiteSettings < ActiveRecord::Migration[8.0]+ def up+ execute <<~SQL+ DELETE FROM site_settings+ WHERE name IN ('persist_browser_pageview_events', 'trigger_browser_pageview_events')+ SQL+ end++ def down+ raise ActiveRecord::IrreversibleMigration+ end+enddiff --git a/db/migrate/20260910030345_create_browser_pageview_event_backups.rb b/db/migrate/20260910030345_create_browser_pageview_event_backups.rbnew file mode 100644index 00000000000..496cde7e8a9--- /dev/null+++ b/db/migrate/20260910030345_create_browser_pageview_event_backups.rb@@ -0,0 +1,28 @@+# frozen_string_literal: true++class CreateBrowserPageviewEventBackups < ActiveRecord::Migration[8.0]+ def up+ execute <<~SQL+ CREATE TABLE browser_pageview_events_backup (LIKE browser_pageview_events);+ ALTER TABLE browser_pageview_events_backup ADD PRIMARY KEY (id);++ CREATE TABLE browser_pageview_event_scores_backup (LIKE browser_pageview_event_scores);+ ALTER TABLE browser_pageview_event_scores_backup ADD PRIMARY KEY (id);++ CREATE OR REPLACE FUNCTION discourse_functions.skip_piggyback_browser_pageview_events()+ RETURNS trigger AS $$+ BEGIN+ IF NEW.source = 1 THEN+ RETURN NULL;+ END IF;++ RETURN NEW;+ END;+ $$ LANGUAGE plpgsql;+ SQL+ end++ def down+ raise ActiveRecord::IrreversibleMigration+ end+enddiff --git a/db/migrate/20260910030404_archive_duplicated_legacy_browser_pageview_events.rb b/db/migrate/20260910030404_archive_duplicated_legacy_browser_pageview_events.rbnew file mode 100644index 00000000000..4d833d6af09--- /dev/null+++ b/db/migrate/20260910030404_archive_duplicated_legacy_browser_pageview_events.rb@@ -0,0 +1,64 @@+# frozen_string_literal: true++class ArchiveDuplicatedLegacyBrowserPageviewEvents < ActiveRecord::Migration[8.0]+ disable_ddl_transaction!++ BATCH_SIZE = 10_000++ def up+ last_legacy_id = 0++ loop do+ last_processed_id = DB.query_single(<<~SQL, batch_size: BATCH_SIZE, last_legacy_id:).first+ WITH legacy_events_batch AS MATERIALIZED (+ SELECT id, session_id+ FROM browser_pageview_events+ WHERE source = 1+ AND id > :last_legacy_id+ ORDER BY id+ LIMIT :batch_size+ ),+ duplicated_legacy_events AS MATERIALIZED (+ SELECT legacy.id+ FROM legacy_events_batch legacy+ WHERE EXISTS (+ SELECT 1+ FROM browser_pageview_events beacon+ WHERE beacon.source = 2+ AND beacon.session_id = legacy.session_id+ )+ LIMIT :batch_size+ ),+ deleted_scores AS (+ DELETE FROM browser_pageview_event_scores scores+ USING duplicated_legacy_events+ WHERE scores.event_id = duplicated_legacy_events.id+ RETURNING scores.*+ ),+ archived_scores AS (+ INSERT INTO browser_pageview_event_scores_backup+ SELECT * FROM deleted_scores+ ),+ deleted_events AS (+ DELETE FROM browser_pageview_events events+ USING duplicated_legacy_events+ WHERE events.id = duplicated_legacy_events.id+ RETURNING events.*+ ),+ archived_events AS (+ INSERT INTO browser_pageview_events_backup+ SELECT * FROM deleted_events+ )+ SELECT MAX(id)+ FROM legacy_events_batch+ SQL++ break if last_processed_id.nil?+ last_legacy_id = last_processed_id+ end+ end++ def down+ raise ActiveRecord::IrreversibleMigration+ end+enddiff --git a/db/migrate/20260910030427_archive_piggyback_browser_pageviews_after_beacon_cutover.rb b/db/migrate/20260910030427_archive_piggyback_browser_pageviews_after_beacon_cutover.rbnew file mode 100644index 00000000000..d751546216c--- /dev/null+++ b/db/migrate/20260910030427_archive_piggyback_browser_pageviews_after_beacon_cutover.rb@@ -0,0 +1,78 @@+# frozen_string_literal: true++class ArchivePiggybackBrowserPageviewsAfterBeaconCutover < ActiveRecord::Migration[8.0]+ disable_ddl_transaction!++ BATCH_SIZE = 10_000++ def up+ # Match ApplicationRequest's daily source selection without loading application code.+ cutover_date = DB.query_single(<<~SQL).first+ WITH first_beacon AS (+ SELECT MIN(date) AS date+ FROM application_requests+ WHERE req_type IN (17, 19) AND count > 0+ )+ SELECT CASE WHEN EXISTS (+ SELECT 1 FROM application_requests legacy+ WHERE legacy.date = first_beacon.date+ AND legacy.req_type IN (13, 15) AND legacy.count > 0+ ) THEN first_beacon.date + 1 ELSE first_beacon.date END+ FROM first_beacon+ SQL+ puts(+ if cutover_date.nil?+ "No cutover date found, skipping migration"+ else+ "Cutover date: #{cutover_date}"+ end,+ )+ return if cutover_date.nil?++ last_legacy_id = 0++ loop do+ last_processed_id =+ DB.query_single(<<~SQL, batch_size: BATCH_SIZE, last_legacy_id:, cutover_date:).first+ WITH legacy_events_batch AS MATERIALIZED (+ SELECT id+ FROM browser_pageview_events+ WHERE source = 1+ AND created_at >= :cutover_date+ AND id > :last_legacy_id+ ORDER BY id+ LIMIT :batch_size+ ),+ deleted_scores AS (+ DELETE FROM browser_pageview_event_scores scores+ USING legacy_events_batch+ WHERE scores.event_id = legacy_events_batch.id+ RETURNING scores.*+ ),+ archived_scores AS (+ INSERT INTO browser_pageview_event_scores_backup+ SELECT * FROM deleted_scores+ ),+ deleted_events AS (+ DELETE FROM browser_pageview_events events+ USING legacy_events_batch+ WHERE events.id = legacy_events_batch.id+ RETURNING events.*+ ),+ archived_events AS (+ INSERT INTO browser_pageview_events_backup+ SELECT * FROM deleted_events+ )+ SELECT MAX(id)+ FROM legacy_events_batch+ SQL++ break if last_processed_id.nil?+ last_legacy_id = last_processed_id+ end+ end++ def down+ raise ActiveRecord::IrreversibleMigration+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex 1e72f1d5972..5c7f093265a 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -158,6 +158,23 @@ CREATE FUNCTION discourse_functions.raise_topic_timers_topic_id_readonly() RETUR$$;+--+-- Name: skip_piggyback_browser_pageview_events(); Type: FUNCTION; Schema: discourse_functions; Owner: -+--++CREATE FUNCTION discourse_functions.skip_piggyback_browser_pageview_events() RETURNS trigger+ LANGUAGE plpgsql+ AS $$+BEGIN+ IF NEW.source = 1 THEN+ RETURN NULL;+ END IF;++ RETURN NEW;+END;+$$;++SET default_tablespace = '';SET default_table_access_method = heap;@@ -2114,6 +2131,27 @@ CREATE TABLE public.browser_pageview_event_scores ();+--+-- Name: browser_pageview_event_scores_backup; Type: TABLE; Schema: public; Owner: -+--++CREATE TABLE public.browser_pageview_event_scores_backup (+ id bigint NOT NULL,+ event_id bigint NOT NULL,+ automation_ua_score smallint NOT NULL,+ known_asn_score smallint NOT NULL,+ velocity_score smallint NOT NULL,+ churn_score smallint NOT NULL,+ rapid_nav_score smallint NOT NULL,+ referrer_score smallint NOT NULL,+ engagement_score smallint NOT NULL,+ ip_rotation_score smallint NOT NULL,+ datacenter_asn_score smallint NOT NULL,+ single_request_no_referrer_score smallint NOT NULL,+ stale_browser_score smallint NOT NULL+);++---- Name: browser_pageview_event_scores_id_seq; Type: SEQUENCE; Schema: public; Owner: ---@@ -2152,7 +2190,35 @@ CREATE TABLE public.browser_pageview_events (score integer,normalized_referrer character varying(2000),normalized_referrer_version smallint,- source smallint DEFAULT 1 NOT NULL,+ source smallint DEFAULT 2 NOT NULL,+ normalized_url character varying(2000),+ normalized_url_version integer,+ browser smallint,+ language character varying(255),+ normalized_language character varying+);+++--+-- Name: browser_pageview_events_backup; Type: TABLE; Schema: public; Owner: -+--++CREATE TABLE public.browser_pageview_events_backup (+ id bigint NOT NULL,+ url character varying(2000) NOT NULL,+ ip_address inet NOT NULL,+ referrer character varying(2000),+ user_agent character varying(1000) NOT NULL,+ session_id character varying(32) NOT NULL,+ topic_id integer,+ user_id integer,+ country_code character varying(2),+ created_at timestamp without time zone NOT NULL,+ asn integer,+ score integer,+ normalized_referrer character varying(2000),+ normalized_referrer_version smallint,+ source smallint NOT NULL,normalized_url character varying(2000),normalized_url_version integer,browser smallint,@@ -16273,6 +16339,14 @@ ALTER TABLE ONLY public.browser_pageview_entry_url_daily_rollupsADD CONSTRAINT browser_pageview_entry_url_daily_rollups_pkey PRIMARY KEY (id);+--+-- Name: browser_pageview_event_scores_backup browser_pageview_event_scores_backup_pkey; Type: CONSTRAINT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.browser_pageview_event_scores_backup+ ADD CONSTRAINT browser_pageview_event_scores_backup_pkey PRIMARY KEY (id);++---- Name: browser_pageview_event_scores browser_pageview_event_scores_pkey; Type: CONSTRAINT; Schema: public; Owner: ---@@ -16281,6 +16355,14 @@ ALTER TABLE ONLY public.browser_pageview_event_scoresADD CONSTRAINT browser_pageview_event_scores_pkey PRIMARY KEY (id);+--+-- Name: browser_pageview_events_backup browser_pageview_events_backup_pkey; Type: CONSTRAINT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.browser_pageview_events_backup+ ADD CONSTRAINT browser_pageview_events_backup_pkey PRIMARY KEY (id);++---- Name: browser_pageview_events browser_pageview_events_pkey; Type: CONSTRAINT; Schema: public; Owner: ---@@ -18986,7 +19068,7 @@ CREATE INDEX idx_bpe_beacon_created_at_id ON public.browser_pageview_events USIN-- Name: idx_bpe_browser_backfill; Type: INDEX; Schema: public; Owner: ----CREATE INDEX idx_bpe_browser_backfill ON public.browser_pageview_events USING btree (source, created_at DESC, id DESC) WHERE (browser IS NULL);+CREATE INDEX idx_bpe_browser_backfill ON public.browser_pageview_events USING btree (created_at DESC, id DESC) WHERE (browser IS NULL);--@@ -18996,6 +19078,13 @@ CREATE INDEX idx_bpe_browser_backfill ON public.browser_pageview_events USING btCREATE INDEX idx_bpe_created_at_country_code ON public.browser_pageview_events USING btree (created_at, country_code);+--+-- Name: idx_bpe_created_at_id; Type: INDEX; Schema: public; Owner: -+--++CREATE INDEX idx_bpe_created_at_id ON public.browser_pageview_events USING btree (created_at DESC, id DESC);++---- Name: idx_bpe_created_at_normalized_referrer; Type: INDEX; Schema: public; Owner: ---@@ -24834,6 +24923,13 @@ CREATE TRIGGER category_settings_require_topic_approval_readonly BEFORE INSERT OCREATE TRIGGER discourse_rss_polling_rss_feeds_author_readonly BEFORE INSERT OR UPDATE OF author ON public.discourse_rss_polling_rss_feeds FOR EACH ROW WHEN ((new.author IS NOT NULL)) EXECUTE FUNCTION discourse_functions.raise_discourse_rss_polling_rss_feeds_author_readonly();+--+-- Name: browser_pageview_events skip_piggyback_browser_pageview_events; Type: TRIGGER; Schema: public; Owner: -+--++CREATE TRIGGER skip_piggyback_browser_pageview_events BEFORE INSERT OR UPDATE OF source ON public.browser_pageview_events FOR EACH ROW WHEN ((new.source = ANY (ARRAY[1, 2]))) EXECUTE FUNCTION discourse_functions.skip_piggyback_browser_pageview_events();++---- Name: topic_timers topic_timers_topic_id_readonly; Type: TRIGGER; Schema: public; Owner: ---@@ -25084,6 +25180,9 @@ INSERT INTO "schema_migrations" (version) VALUES('20260914172801'),('20260914172757'),('20260910033302'),+('20260910030427'),+('20260910030404'),+('20260910030345'),('20260909181443'),('20260908160656'),('20260908153158'),@@ -25094,8 +25193,12 @@ INSERT INTO "schema_migrations" (version) VALUES('20260903195501'),('20260903065141'),('20260902150024'),+('20260902075239'),('20260901020329'),('20260831162602'),+('20260831011840'),+('20260831011839'),+('20260831011836'),('20260828145150'),('20260827064809'),('20260826133816'),diff --git a/frontend/discourse/app/instance-initializers/detect-human-activity.js b/frontend/discourse/app/instance-initializers/detect-human-activity.jsindex 69acb165943..d5dc60f4116 100644--- a/frontend/discourse/app/instance-initializers/detect-human-activity.js+++ b/frontend/discourse/app/instance-initializers/detect-human-activity.js@@ -2,13 +2,6 @@ export default {after: "inject-objects",initialize(owner) {- const enabled =- document.querySelector("meta[name=discourse-engagement-tracking-enabled]")- ?.content === "true";- if (!enabled) {- return;- }-owner.lookup("service:human-activity-tracker").start();},};diff --git a/frontend/discourse/app/instance-initializers/message-bus.js b/frontend/discourse/app/instance-initializers/message-bus.jsindex 21deb31a3c1..b4bb8d74eb8 100644--- a/frontend/discourse/app/instance-initializers/message-bus.js+++ b/frontend/discourse/app/instance-initializers/message-bus.js@@ -8,16 +8,6 @@ import userPresent, { onPresenceChange } from "discourse/lib/user-presence";const LONG_POLL_AFTER_UNSEEN_TIME = 1200000; // 20 minutes-let _sendDeferredPageview = false;-let _deferredURL = null;-let _deferredSessionId = null;-let _deferredReferrer = null;-let _deferredViewTopicId = null;--export function sendDeferredPageview() {- _sendDeferredPageview = true;-}-function mbAjax(messageBus, opts) {opts.headers ||= {};@@ -33,28 +23,6 @@ function mbAjax(messageBus, opts) {opts.headers["Discourse-Present"] = "true";}- if (_sendDeferredPageview) {- opts.headers["Discourse-Track-View-Deferred"] = "true";- if (_deferredSessionId) {- opts.headers["Discourse-Track-View-Session-Id"] = _deferredSessionId;- }- if (_deferredURL) {- opts.headers["Discourse-Track-View-Url"] = _deferredURL;- }- if (_deferredReferrer) {- opts.headers["Discourse-Track-View-Referrer"] = _deferredReferrer;- }- if (_deferredViewTopicId) {- opts.headers["Discourse-Track-View-Topic-Id"] = _deferredViewTopicId;- }-- _sendDeferredPageview = false;- _deferredURL = null;- _deferredSessionId = null;- _deferredReferrer = null;- _deferredViewTopicId = null;- }-const oldComplete = opts.complete;opts.complete = function (xhr, stat) {handleLogoff(xhr) || oldComplete?.(xhr, stat);@@ -74,8 +42,7 @@ export default {const messageBus = owner.lookup("service:message-bus"),user = owner.lookup("service:current-user"),- siteSettings = owner.lookup("service:site-settings"),- router = owner.lookup("service:router");+ siteSettings = owner.lookup("service:site-settings");messageBus.alwaysLongPoll = !isProduction();messageBus.shouldLongPollCallback = () =>@@ -108,22 +75,6 @@ export default {// pass in a positionconst interval = setInterval(() => {if (document.readyState === "complete") {- if (- router.currentRouteName === "topic.fromParams" ||- router.currentRouteName === "topic.fromParamsNear"- ) {- _deferredViewTopicId = router.currentRoute.parent.params.id;- }-- _deferredSessionId = document.querySelector(- "meta[name=discourse-track-view-session-id]"- )?.content;-- if (_deferredSessionId) {- _deferredURL = window.location.href;- _deferredReferrer = document.referrer;- }-clearInterval(interval);messageBus.start();}diff --git a/frontend/discourse/app/instance-initializers/page-tracking.js b/frontend/discourse/app/instance-initializers/page-tracking.jsindex 9c3b102528c..4b138219cad 100644--- a/frontend/discourse/app/instance-initializers/page-tracking.js+++ b/frontend/discourse/app/instance-initializers/page-tracking.js@@ -11,7 +11,6 @@ import {resetPageTracking,startPageTracking,} from "discourse/lib/page-tracker";-import { sendDeferredPageview } from "./message-bus";let _preNavigationUrl = null;@@ -20,21 +19,14 @@ export default {before: "message-bus",initialize(owner) {- const isErrorPage =- document.querySelector("meta#discourse-error")?.dataset.discourseError ===- "true";- if (!isErrorPage) {- sendDeferredPageview();- }-// Tell our AJAX system to track a page transition// eslint-disable-next-line ember/no-private-routing-serviceconst router = owner.lookup("router:main");router.on("routeWillChange", this.handleRouteWillChange);- const dashboardImprovementsEnabled =- document.querySelector("meta[name=discourse-beacon-pageview-enabled]")- ?.content === "true";- if (dashboardImprovementsEnabled) {+ const isErrorPage =+ document.querySelector("meta#discourse-error")?.dataset.discourseError ===+ "true";+ if (!isErrorPage) {router.on("routeDidChange", this.handleRouteDidChange);}@@ -135,7 +127,7 @@ export default {},handleRouteWillChange(transition) {- // transition.from will be null on initial boot transition, which is already tracked as a pageview via the HTML request+ // The initial transition is tracked by handleRouteDidChange.if (!transition.from) {return;}diff --git a/frontend/discourse/app/lib/beacon-pageview.js b/frontend/discourse/app/lib/beacon-pageview.jsindex cecdfbaedf5..116ceb0077f 100644--- a/frontend/discourse/app/lib/beacon-pageview.js+++ b/frontend/discourse/app/lib/beacon-pageview.js@@ -8,6 +8,10 @@ export function hasPendingBeaconRequests() {}export function sendBeaconPageview({ sessionId, url, referrer, topicId }) {+ if (!sessionId) {+ return;+ }+const body = {session_id: sessionId,url,diff --git a/frontend/discourse/scripts/js/pageview.js b/frontend/discourse/scripts/js/pageview.jsindex 1899bd6b62d..0d31e711628 100644--- a/frontend/discourse/scripts/js/pageview.js+++ b/frontend/discourse/scripts/js/pageview.js@@ -2,51 +2,30 @@ document.addEventListener("DOMContentLoaded", function () {const isErrorPage =document.querySelector("meta#discourse-error")?.dataset.discourseError ==="true";+ if (isErrorPage) {+ return;+ }- if (!isErrorPage) {- const root =- document.querySelector("meta[name=discourse-base-uri]")?.content || "";-- const trackViewSessionId = document.querySelector(- "meta[name=discourse-track-view-session-id]"- )?.content;- if (!trackViewSessionId) {- return;- }-- const useBeacon =- document.querySelector("meta[name=discourse-beacon-pageview-enabled]")- ?.content === "true";-- let headers = {- "Discourse-Track-View-Deferred": "true",- };+ const trackViewSessionId = document.querySelector(+ "meta[name=discourse-track-view-session-id]"+ )?.content;+ if (!trackViewSessionId) {+ return;+ }- if (trackViewSessionId) {- headers = Object.assign(headers, {- "Discourse-Track-View-Url": window.location.href,- "Discourse-Track-View-Referrer": document.referrer,- "Discourse-Track-View-Session-Id": trackViewSessionId,- });- }- fetch(`${root}/pageview`, {- method: "POST",- headers,- });+ const root =+ document.querySelector("meta[name=discourse-base-uri]")?.content || "";+ const body = {+ session_id: trackViewSessionId,+ url: window.location.href,+ referrer: document.referrer.length ? document.referrer : null,+ language: navigator.language,+ };- if (useBeacon) {- const body = {- session_id: trackViewSessionId,- url: window.location.href,- referrer: document.referrer.length ? document.referrer : null,- language: navigator.language,- };- fetch(`${root}/srv/pv`, {- method: "POST",- keepalive: true,- headers: { "Content-Type": "application/json" },- body: JSON.stringify(body),- });- }- }+ fetch(`${root}/srv/pv`, {+ method: "POST",+ keepalive: true,+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify(body),+ });});diff --git a/frontend/discourse/tests/acceptance/page-tracking-test.js b/frontend/discourse/tests/acceptance/page-tracking-test.jsindex b25f9a4d26b..9e9bb931cd3 100644--- a/frontend/discourse/tests/acceptance/page-tracking-test.js+++ b/frontend/discourse/tests/acceptance/page-tracking-test.js@@ -8,6 +8,8 @@ import { acceptance } from "discourse/tests/helpers/qunit-helpers";const trackViewHeaderName = "Discourse-Track-View";function setupPretender(server, helper) {+ server.post("/srv/pv", () => helper.response({}));+server.get("/fake-analytics-endpoint", (request) => {if (request.requestHeaders[trackViewHeaderName]) {throw "Fake analytics endpoint was called with track-view header";@@ -22,13 +24,34 @@ function setupFakeAnalytics(ref) {.on("routeDidChange", () => ajax("/fake-analytics-endpoint"));}-function assertRequests({ assert, tracked, untracked, message }) {+function setupTrackingSessionId(needs) {+ let meta;++ needs.hooks.beforeEach(function () {+ meta = document.createElement("meta");+ meta.name = "discourse-track-view-session-id";+ meta.content = "tracking-session-id";+ document.head.appendChild(meta);++ this.owner.lookup("service:human-activity-tracker").scheduleFlush = () =>+ null;+ });++ needs.hooks.afterEach(function () {+ meta.remove();+ });+}++function assertRequests({ assert, tracked, untracked, beacons, message }) {let trackedCount = 0,- untrackedCount = 0;+ untrackedCount = 0,+ beaconCount = 0;const requests = pretender.handledRequests;requests.forEach((request) => {- if (request.requestHeaders[trackViewHeaderName]) {+ if (request.url === "/srv/pv") {+ beaconCount++;+ } else if (request.requestHeaders[trackViewHeaderName]) {trackedCount++;} else {untrackedCount++;@@ -37,6 +60,7 @@ function assertRequests({ assert, tracked, untracked, message }) {assert.strictEqual(trackedCount, tracked, `${message} (tracked)`);assert.strictEqual(untrackedCount, untracked, `${message} (untracked)`);+ assert.strictEqual(beaconCount, beacons, `${message} (beacons)`);pretender.handledRequests = [];}@@ -44,14 +68,16 @@ function assertRequests({ assert, tracked, untracked, message }) {acceptance("Page tracking - loading slider", function (needs) {needs.user();needs.pretender(setupPretender);+ setupTrackingSessionId(needs);- test("sets the discourse-track-view header correctly", async function (assert) {+ test("sets the Discourse-Track-View header on the navigation request", async function (assert) {setupFakeAnalytics(this);assertRequests({assert,tracked: 0,untracked: 1,+ beacons: 0,message: "no tracked requests before app boot",});@@ -60,6 +86,7 @@ acceptance("Page tracking - loading slider", function (needs) {assert,tracked: 0,untracked: 2,+ beacons: 1,message: "no ajax tracked for initial page load",});@@ -68,6 +95,7 @@ acceptance("Page tracking - loading slider", function (needs) {assert,tracked: 1,untracked: 1,+ beacons: 1,message: "tracked one pageview for reloading latest",});@@ -76,6 +104,7 @@ acceptance("Page tracking - loading slider", function (needs) {assert,tracked: 1,untracked: 1,+ beacons: 1,message: "tracked one pageview for navigating to topic",});});@@ -84,17 +113,19 @@ acceptance("Page tracking - loading slider", function (needs) {acceptance("Page tracking - loading spinner", function (needs) {needs.user();needs.pretender(setupPretender);+ setupTrackingSessionId(needs);needs.settings({page_loading_indicator: "spinner",});- test("sets the discourse-track-view header correctly", async function (assert) {+ test("sets the Discourse-Track-View header on the navigation request", async function (assert) {setupFakeAnalytics(this);assertRequests({assert,tracked: 0,untracked: 1,+ beacons: 0,message: "no tracked requests before app boot",});@@ -103,6 +134,7 @@ acceptance("Page tracking - loading spinner", function (needs) {assert,tracked: 0,untracked: 2,+ beacons: 1,message: "no ajax tracked for initial page load",});@@ -111,6 +143,7 @@ acceptance("Page tracking - loading spinner", function (needs) {assert,tracked: 1,untracked: 1,+ beacons: 1,message: "tracked one pageview for reloading latest",});@@ -119,7 +152,26 @@ acceptance("Page tracking - loading spinner", function (needs) {assert,tracked: 1,untracked: 1,+ beacons: 1,message: "tracked one pageview for navigating to topic",});});});++acceptance("Page tracking - without a session ID", function (needs) {+ needs.user();+ needs.pretender(setupPretender);++ test("does not send beacon requests", async function (assert) {+ await visit("/");++ const beaconRequests = pretender.handledRequests.filter(+ (request) => request.url === "/srv/pv"+ );+ assert.strictEqual(+ beaconRequests.length,+ 0,+ "page tracking skips the beacon request"+ );+ });+});diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rbindex 1fb50682ec0..e776ff532d0 100644--- a/lib/crawler_scorer.rb+++ b/lib/crawler_scorer.rb@@ -117,7 +117,6 @@ class CrawlerScorere.asn,e.url,e.created_at,- e.source,(se.session_id IS NOT NULL) AS engagedFROM browser_pageview_events eLEFT JOIN browser_pageview_session_engagements se@@ -131,7 +130,6 @@ class CrawlerScorerSELECTe.ip_address,e.user_agent,- e.source,AVG(CASE WHEN se.session_id IS NOT NULL THEN 0.0 ELSE 1.0 END)AS no_engagement_ratioFROM browser_pageview_events e@@ -145,19 +143,14 @@ class CrawlerScorerFROM events wWHERE w.ip_address = e.ip_addressAND w.user_agent = e.user_agent- AND w.source = e.source)- GROUP BY e.ip_address, e.user_agent, e.source+ GROUP BY e.ip_address, e.user_agent),- -- Per-heuristic stats are partitioned by source as well as ip/ua so that- -- pageviews recorded through different transports (e.g. piggyback vs- -- beacon) never inflate one another's velocity, churn or navigation gaps.ipua_stats AS (SELECTip_address,user_agent,- source,COUNT(*) AS pageviews,COUNT(DISTINCT session_id) AS distinct_sessions,AVG(@@ -168,16 +161,15 @@ class CrawlerScorerEND) AS bad_referrer_ratioFROM events- GROUP BY ip_address, user_agent, source+ GROUP BY ip_address, user_agent),gaps AS (SELECTip_address,user_agent,- source,EXTRACT(EPOCH FROM created_at - LAG(created_at) OVER (- PARTITION BY ip_address, user_agent, source ORDER BY created_at+ PARTITION BY ip_address, user_agent ORDER BY created_at)) AS gap_secondsFROM events),@@ -186,23 +178,21 @@ class CrawlerScorerSELECTip_address,user_agent,- source,percentile_cont(0.5) WITHIN GROUP (ORDER BY gap_seconds)AS median_gap_seconds,COUNT(*) AS gap_countFROM gapsWHERE gap_seconds IS NOT NULL- GROUP BY ip_address, user_agent, source+ GROUP BY ip_address, user_agent),session_stats AS (SELECTsession_id,- source,COUNT(DISTINCT ip_address) AS distinct_ips,EXTRACT(EPOCH FROM MAX(created_at) - MIN(created_at)) AS span_secondsFROM events- GROUP BY session_id, source+ GROUP BY session_id),breakdown AS (@@ -284,10 +274,10 @@ class CrawlerScorerELSE 0END AS engagement_scoreFROM events e- LEFT JOIN ipua_stats iu USING (ip_address, user_agent, source)- LEFT JOIN ipua_engagement ie USING (ip_address, user_agent, source)- LEFT JOIN median_gap mg USING (ip_address, user_agent, source)- LEFT JOIN session_stats ss USING (session_id, source)+ LEFT JOIN ipua_stats iu USING (ip_address, user_agent)+ LEFT JOIN ipua_engagement ie USING (ip_address, user_agent)+ LEFT JOIN median_gap mg USING (ip_address, user_agent)+ LEFT JOIN session_stats ss USING (session_id)),totals AS (diff --git a/lib/dashboard_rollup_rebuilder.rb b/lib/dashboard_rollup_rebuilder.rbindex 101ba245b75..9a134a8a884 100644--- a/lib/dashboard_rollup_rebuilder.rb+++ b/lib/dashboard_rollup_rebuilder.rb@@ -82,11 +82,7 @@ class DashboardRollupRebuilderenddef earliest_event_date- @earliest_event_date ||=- BrowserPageviewEvent- .where(BrowserPageviewEvent.rollup_source_condition)- .minimum(:created_at)- &.to_date+ @earliest_event_date ||= BrowserPageviewEvent.minimum(:created_at)&.to_dateenddef log(message)diff --git a/lib/discourse_dev/application_request.rb b/lib/discourse_dev/application_request.rbindex e62a79bda0c..da8b29fad7c 100644--- a/lib/discourse_dev/application_request.rb+++ b/lib/discourse_dev/application_request.rb@@ -53,10 +53,10 @@ module DiscourseDevrows << [date,- ::ApplicationRequest.req_types[:page_view_logged_in_browser],+ ::ApplicationRequest.req_types[:page_view_logged_in_browser_beacon],logged_in_browser,]- rows << [date, ::ApplicationRequest.req_types[:page_view_anon_browser], anon_browser]+ rows << [date, ::ApplicationRequest.req_types[:page_view_anon_browser_beacon], anon_browser]rows << [date, ::ApplicationRequest.req_types[:page_view_crawler], crawler]rows << [date, ::ApplicationRequest.req_types[:page_view_logged_in], logged_in_legacy]rows << [date, ::ApplicationRequest.req_types[:page_view_anon], anon_legacy]diff --git a/lib/discourse_dev/browser_pageview_event.rb b/lib/discourse_dev/browser_pageview_event.rbindex 2e7a42516fb..21d57ac9e61 100644--- a/lib/discourse_dev/browser_pageview_event.rb+++ b/lib/discourse_dev/browser_pageview_event.rb@@ -46,13 +46,11 @@ module DiscourseDevraise 'To run this rake task in a production site, set the value of `ALLOW_DEV_POPULATE` environment variable to "1"'end- SiteSetting.persist_browser_pageview_events = true-rows = build_rows::BrowserPageviewEvent.insert_all(rows)mirror_to_application_requests(rows)- puts "Enabled persist_browser_pageview_events and inserted #{rows.size} events."+ puts "Inserted #{rows.size} browser pageview events."rows.sizeend@@ -89,7 +87,8 @@ module DiscourseDevdef mirror_to_application_requests(rows)rows.group_by do |row|- req_type = row[:user_id] ? :page_view_logged_in_browser : :page_view_anon_browser+ req_type =+ row[:user_id] ? :page_view_logged_in_browser_beacon : :page_view_anon_browser_beacon[row[:created_at].to_date, req_type]end.each do |(date, req_type), grouped|diff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rbindex ea372e35d89..004eb857e2a 100644--- a/lib/middleware/request_tracker.rb+++ b/lib/middleware/request_tracker.rb@@ -122,8 +122,7 @@ class Middleware::RequestTrackerif tracks_browser_page_view?(data)if data[:is_embed]- count_embed = beacon_pageviews_enabled? ? data[:is_beacon] : !data[:is_beacon]- ApplicationRequest.increment!(:page_view_embed) if count_embed+ ApplicationRequest.increment!(:page_view_embed) if data[:is_beacon]elsif data[:is_beacon]if data[:has_auth_cookie]ApplicationRequest.increment!(:page_view_logged_in_browser_beacon)@@ -429,7 +428,7 @@ class Middleware::RequestTrackerdef log_later(data, env, request)Scheduler::Defer.later("Track view") doif Discourse.pg_readonly_mode?- self.class.track_browser_pageview(data) if SiteSetting.persist_browser_pageview_events+ self.class.track_browser_pageview(data)elseself.class.log_request(data)self.class.track_browser_pageview(data)@@ -597,27 +596,19 @@ class Middleware::RequestTrackerstatus == 200 && !%w[0 false].include?(env_track_view) && request.get? && !is_ajax_request &&is_html_request- # This Discourse-Deferred-Track-View header is piggybacked on a- # follow-up MessageBus request after a real browser loads up a page- # to avoid bots influencing browser page views when loading HTML- # versions of a page.- #- # See `scripts/pageview.js` and `instance-initializers/page-tracking.js`env_deferred_track_view = env["HTTP_DISCOURSE_TRACK_VIEW_DEFERRED"]- deferred_track_view = %w[1 true].include?(env_deferred_track_view)+ deferred_track_view =+ %w[1 true].include?(env_deferred_track_view) && !is_pageview_tracking_request?(request)# This only indicates that we are tracking a page view of some kind, not# using an API key. In #log_request is where we are determining which# of these count as browser page views.#- # TL;DR -- Explicit and Deferred page views count as browser page views (BPVs),- # explicit and implicit page views count as legacy page views.+ # Explicit and implicit page views count as legacy page views. Explicit,+ # deferred, and beacon page views count as browser page views (BPVs).## If this is true, then the X-Discourse-TrackView header is included in# the response.- #- # If the page view is explicit or deferred, then the X-Discourse-BrowserPageView header- # is included in the response.track_view = !!(explicit_track_view || implicit_track_view)browser_page_view = !!(explicit_track_view || deferred_track_view)@@ -667,13 +658,12 @@ class Middleware::RequestTrackertrueend- def self.beacon_pageviews_enabled?- UpcomingChanges.enabled?(:dashboard_improvements) &&- (SiteSetting.persist_browser_pageview_events || SiteSetting.trigger_browser_pageview_events)+ def self.is_beacon_tracking_request?(request)+ request.post? && request.path == Discourse.beacon_pv_tracking_pathend- def self.is_beacon_tracking_request?(request)- beacon_pageviews_enabled? && request.post? && request.path == Discourse.beacon_pv_tracking_path+ def self.is_pageview_tracking_request?(request)+ request.post? && request.path == "#{Discourse.base_path}/pageview"enddef self.same_origin_request?(request)@@ -689,13 +679,8 @@ class Middleware::RequestTrackerfalseend- def self.is_pageview_tracking_request?(request)- request.post? && request.path == "#{Discourse.base_path}/pageview"- end-def self.is_engagement_tracking_request?(request)- SiteSetting.persist_browser_pageview_events && request.post? &&- request.path == Discourse.engagement_tracking_path+ request.post? && request.path == Discourse.engagement_tracking_pathenddef self.track_session_engagement(env)@@ -755,7 +740,6 @@ class Middleware::RequestTracker{track_view: false,explicit_track_view: false,- deferred_track_view: true,implicit_track_view: false,browser_page_view: true,is_beacon: true,@@ -771,30 +755,13 @@ class Middleware::RequestTrackerprivate_class_method :extract_beacon_view_tracking_datadef self.track_browser_pageview(data)+ return if !data[:is_beacon]return if !tracks_browser_page_view?(data)- if !SiteSetting.persist_browser_pageview_events && !SiteSetting.trigger_browser_pageview_events- return- endpayload = build_browser_pageview_event_payload(data)-- persist_browser_pageview_event(payload) if SiteSetting.persist_browser_pageview_events-- if data[:is_beacon]- trigger_beacon_browser_pageview_event(payload)- else- trigger_browser_pageview_event(payload)- end+ persist_browser_pageview_event(payload)end- def self.trigger_browser_pageview_event(payload)- return if SiteSetting.persist_browser_pageview_events- return if !SiteSetting.trigger_browser_pageview_events-- DiscourseEvent.trigger(:browser_pageview, payload)- end- private_class_method :trigger_browser_pageview_event-def self.persist_browser_pageview_event(payload)if REQUIRED_BROWSER_PAGEVIEW_EVENT_FIELDS.any? { |key| payload[key].blank? }Rails.logger.debug("Discarding BrowserPageviewEvent: incomplete payload")@@ -834,15 +801,6 @@ class Middleware::RequestTrackerendprivate_class_method :queue_browser_pageview_event- def self.trigger_beacon_browser_pageview_event(payload)- return if !UpcomingChanges.enabled?(:dashboard_improvements)- return if SiteSetting.persist_browser_pageview_events- return if !SiteSetting.trigger_browser_pageview_events-- DiscourseEvent.trigger(:beacon_browser_pageview, payload)- end- private_class_method :trigger_beacon_browser_pageview_event-def self.build_browser_pageview_event_payload(data)ip_info = DiscourseIpInfo.get(data[:request_remote_ip]){@@ -857,14 +815,6 @@ class Middleware::RequestTrackersession_id: data[:tracking_session_id],topic_id: data[:topic_id],occurred_at: data[:occurred_at],- source:- (- if data[:is_beacon]- BrowserPageviewEvent::SOURCE_BEACON- else- BrowserPageviewEvent::SOURCE_PIGGYBACK- end- ),}endprivate_class_method :build_browser_pageview_event_payload@@ -872,25 +822,17 @@ class Middleware::RequestTrackerprivatedef instrument_browser_page_view(env, request, data)- return unless data[:browser_page_view]+ return unless data[:is_beacon]return unless DiscourseLograge.enabled?return unless self.class.bpv_notifications_enabledrequest ||= Rack::Request.new(env)- if data[:is_beacon]- action = "beacon"- path = request.fullpath- else- action = "piggyback"- path = "#{Discourse.base_path}/pageview"- end-payload = {controller: "PageviewController",- action: action,+ action: "beacon",method: "POST",- path: path,+ path: request.fullpath,format: :json,status: 204,params: {diff --git a/lib/statistics.rb b/lib/statistics.rbindex 17d57581134..a98626f27b0 100644--- a/lib/statistics.rb+++ b/lib/statistics.rb@@ -93,13 +93,13 @@ class Statisticsperiods.map do |(period, key)|anon_page_views =- ApplicationRequest.request_type_count_for_period(:page_view_anon_browser, period)+ ApplicationRequest.browser_pageview_count_for_period(:page_view_anon_browser, period)logged_in_visitors = logged_in_visitors_count(period)next key, anon_page_views if logged_in_visitors == 0logged_in_page_views =- ApplicationRequest.request_type_count_for_period(:page_view_logged_in_browser, period)+ ApplicationRequest.browser_pageview_count_for_period(:page_view_logged_in_browser, period)next key, anon_page_views + logged_in_visitors if logged_in_page_views == 0total_visitors = logged_in_visitors@@ -117,9 +117,9 @@ class Statisticsperiods.map do |(period, key)|logged_in_page_views =- ApplicationRequest.request_type_count_for_period(:page_view_logged_in_browser, period)+ ApplicationRequest.browser_pageview_count_for_period(:page_view_logged_in_browser, period)anon_page_views =- ApplicationRequest.request_type_count_for_period(:page_view_anon_browser, period)+ ApplicationRequest.browser_pageview_count_for_period(:page_view_anon_browser, period)all_logged_in_visitors = logged_in_visitors_count(period)eu_logged_in_visitors = eu_logged_in_visitors_count(period)diff --git a/lib/tasks/populate.rake b/lib/tasks/populate.rakeindex badcf5108dd..e187ca53b99 100644--- a/lib/tasks/populate.rake+++ b/lib/tasks/populate.rake@@ -70,7 +70,7 @@ task "admin_dashboard:populate" => ["db:load_config"] do |_, args|DiscourseDev::ApplicationRequest.populate!end-desc "Enables persist_browser_pageview_events and seeds matching events + application_requests"+desc "Seeds browser pageview events and matching application requests"task "browser_pageview_events:populate", [:count] => ["db:load_config"] do |_, args|DiscourseDev::BrowserPageviewEvent.populate!(count: args[:count]&.to_i)enddiff --git a/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb b/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rbindex c09e8de8437..def6bedda48 100644--- a/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb+++ b/plugins/discourse-ai/lib/admin_dashboard/admin_dashboard_facts.rb@@ -463,7 +463,6 @@ module DiscourseAiAND e.created_at < (:end_date::date + 1)AND e.topic_id IS NOT NULLAND e.normalized_referrer IS NOT NULL- AND #{BrowserPageviewEvent.rollup_source_condition(table: "e")}AND #{topic_conditions}GROUP BY e.topic_id, t.titleORDER BY visits DESCdiff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rbindex 7d46d3fe13b..37ed33d6e6d 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb@@ -252,19 +252,19 @@ module DiscourseDataExplorerid: -42,name: "Crawler and Bot Traffic Overview",description:- "Buckets recent beacon pageviews by bot-likelihood score from Discourse's built-in crawler detection. Scoring lags live traffic, so the newest pageviews sit in the 'not scored' bucket alongside those that carried no bot signals at all, rather than counting as users. WARNING: requires browser pageview event collection (hidden site setting persist_browser_pageview_events) and scoring (hidden site setting experimental_detect_crawler_pageviews); without both, every pageview lands in 'not scored'. Accepts an 'hours' parameter, defaults to the last 24 hours.",+ "Buckets recent beacon pageviews by bot-likelihood score from Discourse's built-in crawler detection. Scoring lags live traffic, so the newest pageviews sit in the 'not scored' bucket alongside those that carried no bot signals at all, rather than counting as users. WARNING: requires scoring (hidden site setting experimental_detect_crawler_pageviews); without it, every pageview lands in 'not scored'. Accepts an 'hours' parameter, defaults to the last 24 hours.",},"crawler-traffic-detailed": {id: -43,name: "Crawler and Bot Traffic Detailed Report",description:- "Row-per-IP breakdown of likely bot pageview activity, with the individual signals that drove the score (automated user agent, known crawler network, velocity, session churn, rapid navigation, bad referrer, no measured interaction). WARNING: requires browser pageview event collection (hidden site setting persist_browser_pageview_events) and scoring (hidden site setting experimental_detect_crawler_pageviews); without both this report will be empty. Accepts 'hours' and 'min_score' parameters.",+ "Row-per-IP breakdown of likely bot pageview activity, with the individual signals that drove the score (automated user agent, known crawler network, velocity, session churn, rapid navigation, bad referrer, no measured interaction). WARNING: requires scoring (hidden site setting experimental_detect_crawler_pageviews); without it this report will be empty. Accepts 'hours' and 'min_score' parameters.",},"suspected-bot-networks": {id: -44,name: "Suspected Automated Traffic by IP and Network",description:- "Networks (ASNs) and IPs generating high pageview volume with bot-like session patterns (near 1.0 views per session, rotating user agents, systematic topic harvesting), sorted so a scrape spread across many IPs on one network floats to the top. WARNING: requires browser pageview event collection to be enabled (hidden site setting persist_browser_pageview_events); without it no events are recorded and this report will be empty. Accepts a 'days_ago' parameter, defaults to the last 3 days.",+ "Networks (ASNs) and IPs generating high pageview volume with bot-like session patterns (near 1.0 views per session, rotating user agents, systematic topic harvesting), sorted so a scrape spread across many IPs on one network floats to the top. Accepts a 'days_ago' parameter, defaults to the last 3 days.",},"ask-ai-asks-over-time": {id: -45,@@ -1528,7 +1528,6 @@ module DiscourseDataExplorerSELECT scoreFROM browser_pageview_eventsWHERE created_at >= NOW() - (:hours * INTERVAL '1 hour')- AND source = #{BrowserPageviewEvent::SOURCE_BEACON})SELECT 'Not scored (pending or no signals)' AS bucket, COUNT(*) FILTER (WHERE score IS NULL) AS pageviews FROM eventsUNION ALL@@ -1586,7 +1585,6 @@ module DiscourseDataExplorerFROM browser_pageview_events eJOIN browser_pageview_event_scores s ON s.event_id = e.idWHERE e.created_at >= NOW() - (:hours * INTERVAL '1 hour')- AND e.source = #{BrowserPageviewEvent::SOURCE_BEACON}AND e.score > :min_scoreGROUP BY e.ip_address, e.user_agent, e.asn, e.country_code, e.user_id, e.session_idORDER BY max_score DESC, pageviews DESC@@ -1613,7 +1611,6 @@ module DiscourseDataExplorer(array_agg(user_agent ORDER BY created_at DESC))[1] AS sample_user_agentFROM browser_pageview_eventsWHERE created_at >= CURRENT_DATE - CAST(:days_ago AS integer)- AND source = #{BrowserPageviewEvent::SOURCE_BEACON}GROUP BY ip_address, asn, country_codeORDER BY asn_total_pageviews DESC, pageviews DESCLIMIT 100diff --git a/spec/helpers/application_helper_spec.rb b/spec/helpers/application_helper_spec.rbindex 41332df2b57..5e150ce7091 100644--- a/spec/helpers/application_helper_spec.rb+++ b/spec/helpers/application_helper_spec.rb@@ -3,23 +3,10 @@RSpec.describe ApplicationHelper dodescribe "#discourse_pageview_tracking_meta_tags" do- it "includes beacon tracking meta tags for anonymous users when dashboard_improvements is enabled" do- SiteSetting.dashboard_improvements = true- helper.stubs(:current_user).returns(nil)-- tags = helper.discourse_pageview_tracking_meta_tags-- expect(tags).to include('name="discourse-track-view-session-id"')- expect(tags).to include('name="discourse-beacon-pageview-enabled"')- end-- it "omits beacon tracking meta tags when dashboard_improvements is disabled" do- SiteSetting.dashboard_improvements = false-+ it "includes pageview tracking metadata" dotags = helper.discourse_pageview_tracking_meta_tagsexpect(tags).to include('name="discourse-track-view-session-id"')- expect(tags).not_to include('name="discourse-beacon-pageview-enabled"')endit "omits tracking meta tags in development when request tracking is not enabled" do@@ -32,36 +19,6 @@ RSpec.describe ApplicationHelper doensureENV["TRACK_REQUESTS"] = original_track_requests if original_track_requestsend-- it "includes beacon tracking meta tags for browser pageview event triggers" do- SiteSetting.dashboard_improvements = true- SiteSetting.persist_browser_pageview_events = false- SiteSetting.trigger_browser_pageview_events = true-- tags = helper.discourse_pageview_tracking_meta_tags-- expect(tags).to include('name="discourse-track-view-session-id"')- expect(tags).to include('name="discourse-beacon-pageview-enabled"')- end-- it "includes the engagement tracking meta tag when pageview events are persisted" do- SiteSetting.persist_browser_pageview_events = true-- tags = helper.discourse_pageview_tracking_meta_tags-- expect(tags).to include('name="discourse-engagement-tracking-enabled"')- end-- it "omits the engagement tracking meta tag in a trigger-only config so the browser does not send /srv/se beacons the server rejects" do- SiteSetting.dashboard_improvements = true- SiteSetting.persist_browser_pageview_events = false- SiteSetting.trigger_browser_pageview_events = true-- tags = helper.discourse_pageview_tracking_meta_tags-- expect(tags).to include('name="discourse-beacon-pageview-enabled"')- expect(tags).not_to include('name="discourse-engagement-tracking-enabled"')- endenddescribe "preload_script" dodiff --git a/spec/jobs/flush_browser_pageview_events_spec.rb b/spec/jobs/flush_browser_pageview_events_spec.rbindex a111e242d37..772b3dd3fea 100644--- a/spec/jobs/flush_browser_pageview_events_spec.rb+++ b/spec/jobs/flush_browser_pageview_events_spec.rb@@ -17,7 +17,6 @@ RSpec.describe Jobs::FlushBrowserPageviewEvents doip_address: "1.2.3.4",user_agent: "Mozilla/5.0",session_id: "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx",- source: BrowserPageviewEvent::SOURCE_BEACON,occurred_at: Time.zone.parse("2026-05-27 10:30:00").iso8601(6),}end@@ -30,17 +29,7 @@ RSpec.describe Jobs::FlushBrowserPageviewEvents dopayloads.each { |queued_payload| queue_payload(queued_payload) }end- it "does nothing when browser pageview persistence is disabled" do- SiteSetting.persist_browser_pageview_events = false- queue_payload(payload)-- expect { described_class.new.execute({}) }.not_to change { BrowserPageviewEvent.count }-- expect(BrowserPageviewEvent.queued_count).to eq(1)- end-it "flushes queued browser pageviews" do- SiteSetting.persist_browser_pageview_events = truequeue_payload(payload)expect { described_class.new.execute({}) }.to change { BrowserPageviewEvent.count }.by(1)@@ -49,8 +38,6 @@ RSpec.describe Jobs::FlushBrowserPageviewEvents doendit "flushes multiple batches in one run" do- SiteSetting.persist_browser_pageview_events = true-stub_const(BrowserPageviewEvent, "REDIS_FLUSH_BATCH_SIZE", 3) dopayloads = 5.times.map { |index| payload.merge(session_id: format("%032d", index)) }queue_payloads(payloads)diff --git a/spec/jobs/maintain_browser_pageview_rollups_spec.rb b/spec/jobs/maintain_browser_pageview_rollups_spec.rbindex 4067b246008..4fdad02a016 100644--- a/spec/jobs/maintain_browser_pageview_rollups_spec.rb+++ b/spec/jobs/maintain_browser_pageview_rollups_spec.rb@@ -3,25 +3,7 @@RSpec.describe Jobs::MaintainBrowserPageviewRollups dosubject(:job) { described_class.new }- before { SiteSetting.persist_browser_pageview_events = true }-describe "#execute" do- it "does nothing when persist_browser_pageview_events is disabled" do- SiteSetting.persist_browser_pageview_events = false- event =- Fabricate(- :browser_pageview_event_with_unnormalized_referrer,- referrer: "https://reddit.com/",- country_code: "US",- )-- job.execute({})-- expect(BrowserPageviewCountryDailyRollup.count).to eq(0)- expect(BrowserPageviewReferrerDailyRollup.count).to eq(0)- expect(event.reload.normalized_referrer_version).to be_nil- end-context "when aggregating rollups" doit "aggregates both country and referrer rollups from recent pageview events" doFabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")@@ -34,55 +16,6 @@ RSpec.describe Jobs::MaintainBrowserPageviewRollups do).to eq(1)end- it "rolls up piggyback events before the beacon cutover date and beacon events after it" do- freeze_time(Time.utc(2026, 6, 20, 12))- SiteSetting.dashboard_improvements = true- UpcomingChangeEvent.create!(- upcoming_change_name: "dashboard_improvements",- event_type: :manual_opt_in,- created_at: Time.utc(2026, 6, 10, 9),- )-- Fabricate(- :browser_pageview_event,- country_code: "US",- normalized_referrer: "google.com",- source: :piggyback,- created_at: Time.utc(2026, 6, 9, 10),- )- Fabricate(- :browser_pageview_event,- country_code: "GB",- normalized_referrer: "reddit.com",- source: :beacon,- created_at: Time.utc(2026, 6, 9, 10),- )- Fabricate(- :browser_pageview_event,- country_code: "FR",- normalized_referrer: "bing.com",- source: :beacon,- created_at: Time.utc(2026, 6, 15, 10),- )- Fabricate(- :browser_pageview_event,- country_code: "DE",- normalized_referrer: "duckduckgo.com",- source: :piggyback,- created_at: Time.utc(2026, 6, 15, 10),- )-- job.execute({})-- expect(BrowserPageviewCountryDailyRollup.pluck(:country_code, :count)).to contain_exactly(- ["US", 1],- ["FR", 1],- )- expect(- BrowserPageviewReferrerDailyRollup.pluck(:normalized_referrer, :count),- ).to contain_exactly(["google.com", 1], ["bing.com", 1])- end-it "backfills from the earliest event date on the first run when rollups are empty" doFabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)Fabricate(:browser_pageview_event, country_code: "GB", created_at: 5.days.ago)@@ -177,16 +110,6 @@ RSpec.describe Jobs::MaintainBrowserPageviewRollups docontext "when aggregating engagement rollups" dobefore { freeze_time(Time.utc(2026, 6, 20, 12, 0, 0)) }- it "does nothing when persist_browser_pageview_events is disabled" do- SiteSetting.persist_browser_pageview_events = false- Fabricate(:browser_pageview_session_engagement, created_at: Time.utc(2026, 6, 10))- Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 10, 9))-- job.execute({})-- expect(BrowserPageviewSessionEngagementDailyRollup.count).to eq(0)- end-it "aggregates nothing until the first engagement row exists" doFabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 10, 9))@@ -264,47 +187,6 @@ RSpec.describe Jobs::MaintainBrowserPageviewRollups do)end- it "only backfills referrers from the source that applies on the event's date" do- freeze_time(Time.utc(2026, 6, 20, 12))- SiteSetting.dashboard_improvements = true- UpcomingChangeEvent.create!(- upcoming_change_name: "dashboard_improvements",- event_type: :manual_opt_in,- created_at: Time.utc(2026, 6, 10, 9),- )-- pre_cutover_piggyback =- Fabricate(- :browser_pageview_event_with_unnormalized_referrer,- referrer: "https://www.google.com/",- source: :piggyback,- created_at: Time.utc(2026, 6, 9, 10),- )- post_cutover_piggyback =- Fabricate(- :browser_pageview_event_with_unnormalized_referrer,- referrer: "https://www.bing.com/",- source: :piggyback,- created_at: Time.utc(2026, 6, 15, 10),- )- post_cutover_beacon =- Fabricate(- :browser_pageview_event_with_unnormalized_referrer,- referrer: "https://www.reddit.com/",- source: :beacon,- created_at: Time.utc(2026, 6, 15, 10),- )-- job.execute({})-- expect(pre_cutover_piggyback.reload.normalized_referrer).to eq("google.com")- expect(post_cutover_piggyback.reload.normalized_referrer_version).to be_nil- expect(post_cutover_beacon.reload.normalized_referrer).to eq("reddit.com")- expect(post_cutover_beacon.normalized_referrer_version).to eq(- BrowserPageviewEventUrlNormalizer::REFERRER_VERSION,- )- end-it "leaves rows without a referrer untouched and does not let them block completion" dodirect_visit = Fabricate(:browser_pageview_event_with_unnormalized_referrer, referrer: nil)referred =diff --git a/spec/lib/crawler_scorer_spec.rb b/spec/lib/crawler_scorer_spec.rbindex b7290246610..2414a849543 100644--- a/spec/lib/crawler_scorer_spec.rb+++ b/spec/lib/crawler_scorer_spec.rb@@ -466,40 +466,4 @@ RSpec.describe CrawlerScorer do.engagement_score,).to eq(20)end-- it "scores each source but partitions velocity so transports do not inflate each other" do- stub_const(CrawlerScorer, :VELOCITY_LOW, 10) do- stub_const(CrawlerScorer, :VELOCITY_MEDIUM, 20) do- base = 30.minutes.ago-- # Same ip+ua, split across two transports with 12 pageviews each. On- # its own each source sits in the LOW velocity tier (+15). Combined they- # would be 24 pageviews and reach the MEDIUM tier (+20), so equal- # per-source scores prove the heuristics stay partitioned by source.- {- BrowserPageviewEvent::SOURCE_PIGGYBACK => "piggyback-session",- BrowserPageviewEvent::SOURCE_BEACON => "beacon-session",- }.each do |source, session_id|- 12.times do |i|- make_event(source: source, session_id: session_id, created_at: base + (i * 15).seconds)- end- end-- score!-- expect(- BrowserPageviewEvent- .where(source: BrowserPageviewEvent::SOURCE_PIGGYBACK)- .pluck(:score)- .uniq,- ).to contain_exactly(55)- expect(- BrowserPageviewEvent- .where(source: BrowserPageviewEvent::SOURCE_BEACON)- .pluck(:score)- .uniq,- ).to contain_exactly(55)- end- end- endenddiff --git a/spec/lib/middleware/request_tracker_spec.rb b/spec/lib/middleware/request_tracker_spec.rbindex bf4982d721e..4386c4c2639 100644--- a/spec/lib/middleware/request_tracker_spec.rb+++ b/spec/lib/middleware/request_tracker_spec.rb@@ -15,10 +15,10 @@ RSpec.describe Middleware::RequestTracker doendbefore do+ ApplicationRequest.delete_allApplicationRequest.enableCachedCounting.resetCachedCounting.enable- SiteSetting.persist_browser_pageview_events = falseendafter do@@ -98,7 +98,7 @@ RSpec.describe Middleware::RequestTracker doMiddleware::RequestTracker.log_request(data)end- it "can exclude/include based on custom header" do+ it "counts explicit views in legacy pageview counters" dolog_tracked_view("true")log_tracked_view("1")log_tracked_view("false")@@ -131,33 +131,6 @@ RSpec.describe Middleware::RequestTracker doexpect(headers["X-Discourse-BrowserPageView"]).to eq(nil)end- it "adds the appropriate response header based on deferred tracking (MiniProfiler piggyback, BPVs)" do- middleware = Middleware::RequestTracker.new(lambda { |env| [200, {}, ["OK"]] })- status, headers = middleware.call(env("HTTP_DISCOURSE_TRACK_VIEW_DEFERRED" => "1"))-- expect(status).to eq(200)- expect(headers["X-Discourse-TrackView"]).to eq(nil)- expect(headers["X-Discourse-BrowserPageView"]).to eq("1")- end-- it "adds the appropriate response headers for MessageBus requests with deferred tracking" do- app =- lambda do |env|- headers = MessageBus.extra_response_headers_lookup.call(env)- [200, headers, ["OK"]]- end-- middleware = Middleware::RequestTracker.new(app)-- status, headers =- middleware.call(- env("HTTP_DISCOURSE_TRACK_VIEW_DEFERRED" => "1", :path => "/message-bus/abcde/poll"),- )-- expect(status).to eq(200)- expect(headers["X-Discourse-BrowserPageView"]).to eq("1")- end-it "adds the appropriate response headers for MessageBus requests with regular tracking" doapp =lambda do |env|@@ -253,7 +226,7 @@ RSpec.describe Middleware::RequestTracker doexpect(ApplicationRequest.page_view_anon_browser.first.count).to eq(1)end- it "logs deferred pageviews correctly" do+ it "logs deferred pageviews in legacy browser counters" dodata =Middleware::RequestTracker.get_data(env(:path => "/message-bus/abcde/poll", "HTTP_DISCOURSE_TRACK_VIEW_DEFERRED" => "1"),@@ -261,8 +234,6 @@ RSpec.describe Middleware::RequestTracker do0.1,)Middleware::RequestTracker.log_request(data)-- expect(data[:deferred_track_view]).to eq(true)CachedCounting.flushexpect(ApplicationRequest.page_view_anon_browser.first.count).to eq(1)@@ -283,25 +254,7 @@ RSpec.describe Middleware::RequestTracker doexpect(ApplicationRequest.page_view_embed.sum(:count)).to eq(0)end- it "counts deferred pageview with embed header as page_view_embed" do- data =- Middleware::RequestTracker.get_data(- env(- :path => "/message-bus/abcde/poll",- "HTTP_DISCOURSE_TRACK_VIEW_DEFERRED" => "1",- "HTTP_DISCOURSE_TRACK_VIEW_EMBED" => "true",- ),- ["200", { "Content-Type" => "text/html" }],- 0.1,- )- Middleware::RequestTracker.log_request(data)- CachedCounting.flush-- expect(ApplicationRequest.page_view_embed.first.count).to eq(1)- expect(ApplicationRequest.page_view_anon_browser.sum(:count)).to eq(0)- end-- it "counts explicit XHR pageview with embed header only once as page_view_embed" do+ it "does not count explicit XHR pageviews with embed headers" dodata =Middleware::RequestTracker.get_data(env("HTTP_DISCOURSE_TRACK_VIEW" => "1", "HTTP_DISCOURSE_TRACK_VIEW_EMBED" => "true"),@@ -311,7 +264,7 @@ RSpec.describe Middleware::RequestTracker doMiddleware::RequestTracker.log_request(data)CachedCounting.flush- expect(ApplicationRequest.page_view_embed.first.count).to eq(1)+ expect(ApplicationRequest.page_view_embed.sum(:count)).to eq(0)expect(ApplicationRequest.page_view_anon.sum(:count)).to eq(0)expect(ApplicationRequest.page_view_anon_browser.sum(:count)).to eq(0)end@@ -331,9 +284,7 @@ RSpec.describe Middleware::RequestTracker doMiddleware::RequestTracker.log_request(data)end- it "counts embed views from the beacon instead of the piggyback when beacons are enabled" do- SiteSetting.dashboard_improvements = true- SiteSetting.trigger_browser_pageview_events = true+ it "counts embed views from the beacon instead of explicit tracking" dobody = {session_id: "abc",url: "https://example.com/t/slug/1",@@ -353,19 +304,19 @@ RSpec.describe Middleware::RequestTracker do)Middleware::RequestTracker.log_request(data)- piggyback_data =+ explicit_data =Middleware::RequestTracker.get_data(env("HTTP_DISCOURSE_TRACK_VIEW" => "1", "HTTP_DISCOURSE_TRACK_VIEW_EMBED" => "true"),["200", {}],0.1,)- Middleware::RequestTracker.log_request(piggyback_data)+ Middleware::RequestTracker.log_request(explicit_data)CachedCounting.flushexpect(data[:is_embed]).to eq(true)- expect(piggyback_data[:is_embed]).to eq(true)+ expect(explicit_data[:is_embed]).to eq(true)expect(ApplicationRequest.page_view_embed.sum(:count)).to eq(1)- expect(ApplicationRequest.page_view_anon_browser_beacon.sum(:count)).to eq(0)+ expect(ApplicationRequest.page_view_anon_browser.sum(:count)).to eq(0)endit "survives requests with a missing or unreadable body" do@@ -494,25 +445,17 @@ RSpec.describe Middleware::RequestTracker do)end- def log_topic_view(authenticated: false, deferred: false)- headers = { "action_dispatch.remote_ip" => "127.0.0.1" }-+ def log_topic_view(authenticated: false)+ headers = {+ "action_dispatch.remote_ip" => "127.0.0.1",+ "HTTP_DISCOURSE_TRACK_VIEW" => "1",+ "HTTP_DISCOURSE_TRACK_VIEW_TOPIC_ID" => topic.id,+ }headers["HTTP_COOKIE"] = "_t=#{auth_cookie};" if authenticated- if deferred- headers["HTTP_DISCOURSE_TRACK_VIEW"] = "1"- headers["HTTP_DISCOURSE_TRACK_VIEW_DEFERRED"] = "1"- headers["HTTP_DISCOURSE_TRACK_VIEW_TOPIC_ID"] = topic.id- path = "/message-bus/abcde/poll"- else- headers["HTTP_DISCOURSE_TRACK_VIEW"] = "1"- headers["HTTP_DISCOURSE_TRACK_VIEW_TOPIC_ID"] = topic.id- path = URI.parse(topic.url).path- end-data =Middleware::RequestTracker.get_data(- env(path: path, **headers),+ env(path: URI.parse(topic.url).path, **headers),["200", { "Content-Type" => "text/html" }],0.1,)@@ -520,85 +463,8 @@ RSpec.describe Middleware::RequestTracker dodataend- it "logs deferred topic views correctly for logged in users" do- data = log_topic_view(authenticated: true, deferred: true)-- expect(data[:topic_id]).to eq(topic.id)- expect(data[:request_remote_ip]).to eq("127.0.0.1")- expect(data[:current_user_id]).to eq(user.id)- CachedCounting.flush-- expect(TopicViewItem.exists?(topic_id: topic.id, user_id: user.id, ip_address: nil)).to eq(- true,- )- expect(- TopicViewStat.exists?(- topic_id: topic.id,- anonymous_views: 0,- logged_in_views: 1,- viewed_at: Time.zone.now.to_date,- ),- ).to eq(true)- end-- it "does not log deferred topic views for topics the user cannot access" do- topic.update!(category: Fabricate(:private_category, group: Fabricate(:group)))- log_topic_view(authenticated: true, deferred: true)- CachedCounting.flush- expect(TopicViewItem.exists?(topic_id: topic.id, user_id: user.id, ip_address: nil)).to eq(- false,- )- expect(- TopicViewStat.exists?(- topic_id: topic.id,- anonymous_views: 0,- logged_in_views: 1,- viewed_at: Time.zone.now.to_date,- ),- ).to eq(false)- end-- it "logs deferred topic views correctly for anonymous" do- data = log_topic_view(authenticated: false, deferred: true)-- expect(data[:topic_id]).to eq(topic.id)- expect(data[:request_remote_ip]).to eq("127.0.0.1")- expect(data[:current_user_id]).to eq(nil)- CachedCounting.flush-- expect(- TopicViewItem.exists?(topic_id: topic.id, user_id: nil, ip_address: "127.0.0.1"),- ).to eq(true)- expect(- TopicViewStat.exists?(- topic_id: topic.id,- anonymous_views: 1,- logged_in_views: 0,- viewed_at: Time.zone.now.to_date,- ),- ).to eq(true)- end-- it "does not log deferred topic views for topics the anonymous user cannot access" do- topic.update!(category: Fabricate(:private_category, group: Fabricate(:group)))- log_topic_view(authenticated: false, deferred: true)- CachedCounting.flush-- expect(- TopicViewItem.exists?(topic_id: topic.id, user_id: nil, ip_address: "127.0.0.1"),- ).to eq(false)- expect(- TopicViewStat.exists?(- topic_id: topic.id,- anonymous_views: 1,- logged_in_views: 0,- viewed_at: Time.zone.now.to_date,- ),- ).to eq(false)- end-it "logs explicit topic views correctly for logged in users" do- data = log_topic_view(authenticated: true, deferred: false)+ data = log_topic_view(authenticated: true)expect(data[:topic_id]).to eq(topic.id)expect(data[:request_remote_ip]).to eq("127.0.0.1")@@ -620,7 +486,7 @@ RSpec.describe Middleware::RequestTracker doit "does not log explicit topic views for topics the user cannot access" dotopic.update!(category: Fabricate(:private_category, group: Fabricate(:group)))- log_topic_view(authenticated: true, deferred: false)+ log_topic_view(authenticated: true)CachedCounting.flushexpect(TopicViewItem.exists?(topic_id: topic.id, user_id: user.id, ip_address: nil)).to eq(@@ -637,7 +503,7 @@ RSpec.describe Middleware::RequestTracker doendit "logs explicit topic views correctly for anonymous" do- data = log_topic_view(authenticated: false, deferred: false)+ data = log_topic_viewexpect(data[:topic_id]).to eq(topic.id)expect(data[:request_remote_ip]).to eq("127.0.0.1")@@ -659,7 +525,7 @@ RSpec.describe Middleware::RequestTracker doit "does not log explicit topic views for topics the anonymous user cannot access" dotopic.update!(category: Fabricate(:private_category, group: Fabricate(:group)))- log_topic_view(authenticated: false, deferred: false)+ log_topic_viewCachedCounting.flushexpect(@@ -740,338 +606,67 @@ RSpec.describe Middleware::RequestTracker doendend- describe "browser_pageview event" do+ describe "non-beacon browser pageviews" dodef log_browser_pageview(data)Middleware::RequestTracker.new(lambda { |env| [200, {}, ["OK"]] }).log_later(data, {}, nil)end- context "when SiteSetting.trigger_browser_pageview_events is true" do- before { SiteSetting.trigger_browser_pageview_events = true }-- it "triggers event for anonymous user page views when `login_required` site setting is false" do- session_id = "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx"- DiscourseIpInfo.stubs(:get).returns(country_code: "AU")-- data =- Middleware::RequestTracker.get_data(- env(- "HTTP_DISCOURSE_TRACK_VIEW" => "1",- "HTTP_DISCOURSE_TRACK_VIEW_SESSION_ID" => session_id,- "HTTP_DISCOURSE_TRACK_VIEW_URL" => "https://discourse.org",- "HTTP_DISCOURSE_TRACK_VIEW_REFERRER" => "https://example.com",- ),- ["200", { "Content-Type" => "text/html" }],- 0.2,- )-- events = DiscourseEvent.track_events(:browser_pageview) { log_browser_pageview(data) }-- expect(events.length).to eq(1)- event = events[0][:params].first- expect(event[:user_id]).to be_nil- expect(event[:session_id]).to eq(session_id)- expect(event[:url]).to eq("https://discourse.org")- expect(event[:referrer]).to eq("https://example.com")- expect(event).to have_key(:ip_address)- expect(event[:country_code]).to eq("AU")- expect(event[:user_agent]).to be_present- end-- it "does not trigger event for anonymous user page views when `login_required` site setting is true" do- SiteSetting.login_required = true- session_id = "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx"-- data =- Middleware::RequestTracker.get_data(- env(- "HTTP_DISCOURSE_TRACK_VIEW" => "1",- "HTTP_DISCOURSE_TRACK_VIEW_SESSION_ID" => session_id,- "HTTP_DISCOURSE_TRACK_VIEW_URL" => "https://discourse.org",- "HTTP_DISCOURSE_TRACK_VIEW_REFERRER" => "https://example.com",- ),- ["200", { "Content-Type" => "text/html" }],- 0.2,- )-- events = DiscourseEvent.track_events(:browser_pageview) { log_browser_pageview(data) }-- expect(events).to be_empty- end-- it "truncates session id, url, referrer, ip address and user agent" do- Middleware::AnonymousCache::Helper.any_instance.expects(:is_crawler?).returns(false)- data =- Middleware::RequestTracker.get_data(- env(- "HTTP_DISCOURSE_TRACK_VIEW" => "1",- "HTTP_DISCOURSE_TRACK_VIEW_SESSION_ID" => "A" * 50,- "HTTP_DISCOURSE_TRACK_VIEW_URL" => "A" * 5000,- "HTTP_DISCOURSE_TRACK_VIEW_REFERRER" => "A" * 5000,- "HTTP_USER_AGENT" => "A" * 5000,- "action_dispatch.remote_ip" => "1" * 50,- ),- ["200", { "Content-Type" => "text/html" }],- 0.2,- )-- events = DiscourseEvent.track_events(:browser_pageview) { log_browser_pageview(data) }-- expect(events.length).to eq(1)- event = events[0][:params].first- expect(event[:url].length).to eq(Middleware::RequestTracker::MAX_URL_LENGTH)- expect(event[:referrer].length).to eq(Middleware::RequestTracker::MAX_URL_LENGTH)- expect(event[:session_id].length).to eq(Middleware::RequestTracker::MAX_SESSION_ID_LENGTH)- expect(event[:user_agent].length).to eq(Middleware::RequestTracker::MAX_USER_AGENT_LENGTH)- expect(event[:ip_address].length).to eq(Middleware::RequestTracker::MAX_IP_ADDRESS_LENGTH)- end-- it "triggers event for logged-in user page views" do- user = Fabricate(:user, active: true)- DiscourseIpInfo.stubs(:get).returns(country_code: "DE")- token = UserAuthToken.generate!(user_id: user.id)- cookie =- create_auth_cookie(- token: token.unhashed_auth_token,- user_id: user.id,- trust_level: user.trust_level,- issued_at: 5.minutes.ago,- )-- data =- Middleware::RequestTracker.get_data(- env(- "HTTP_DISCOURSE_TRACK_VIEW" => "1",- "HTTP_COOKIE" => "_t=#{cookie};",- "HTTP_DISCOURSE_TRACK_VIEW_URL" => "https://discourse.org",- "HTTP_DISCOURSE_TRACK_VIEW_REFERRER" => "https://example.com",- ),- ["200", { "Content-Type" => "text/html" }],- 0.2,- )-- events = DiscourseEvent.track_events(:browser_pageview) { log_browser_pageview(data) }-- expect(events.length).to eq(1)- event = events[0][:params].first- expect(event[:user_id]).to eq(user.id)- expect(event[:url]).to eq("https://discourse.org")- expect(event[:referrer]).to eq("https://example.com")- expect(event).to have_key(:ip_address)- expect(event[:country_code]).to eq("DE")- expect(event[:user_agent]).to be_present- end-- it "does not trigger event for crawler page views" do- data =- Middleware::RequestTracker.get_data(- env("HTTP_USER_AGENT" => "Googlebot"),- ["200", { "Content-Type" => "text/html" }],- 0.2,- )-- events = DiscourseEvent.track_events(:browser_pageview) { log_browser_pageview(data) }-- expect(events.length).to eq(0)- end+ let(:pageview_data) do+ Middleware::RequestTracker.get_data(+ env(+ "HTTP_DISCOURSE_TRACK_VIEW" => "1",+ "HTTP_DISCOURSE_TRACK_VIEW_SESSION_ID" => "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx",+ "HTTP_DISCOURSE_TRACK_VIEW_URL" => "https://discourse.org",+ "action_dispatch.remote_ip" => "1.2.3.4",+ ),+ ["200", { "Content-Type" => "text/html" }],+ 0.2,+ )end- context "when SiteSetting.trigger_browser_pageview_events is false" do- it "does not trigger events" do- session_id = "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx"-- data =- Middleware::RequestTracker.get_data(- env(- "HTTP_DISCOURSE_TRACK_VIEW" => "1",- "HTTP_DISCOURSE_TRACK_VIEW_SESSION_ID" => session_id,- "HTTP_DISCOURSE_TRACK_VIEW_URL" => "https://discourse.org",- "HTTP_DISCOURSE_TRACK_VIEW_REFERRER" => "https://example.com",- ),- ["200", { "Content-Type" => "text/html" }],- 0.2,- )-- events = DiscourseEvent.track_events(:browser_pageview) { log_browser_pageview(data) }+ it "does not emit the legacy browser_pageview event" do+ events =+ DiscourseEvent.track_events(:browser_pageview) { log_browser_pageview(pageview_data) }- expect(events.length).to eq(0)- end+ expect(events).to be_emptyend- context "when SiteSetting.persist_browser_pageview_events is true" do- before do- SiteSetting.persist_browser_pageview_events = true- BrowserPageviewEvent.clear_queued!- Discourse.clear_readonly!- end-- after do- BrowserPageviewEvent.clear_queued!- Discourse.clear_readonly!- end-- def flush_browser_pageview_events- Jobs::FlushBrowserPageviewEvents.new.execute({})- end-- it "creates a BrowserPageviewEvent row and does not fire the :browser_pageview event" do- session_id = "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx"- DiscourseIpInfo.stubs(:get).returns(country_code: "AU")-- data =- Middleware::RequestTracker.get_data(- env(- "HTTP_DISCOURSE_TRACK_VIEW" => "1",- "HTTP_DISCOURSE_TRACK_VIEW_SESSION_ID" => session_id,- "HTTP_DISCOURSE_TRACK_VIEW_URL" => "https://discourse.org",- "HTTP_DISCOURSE_TRACK_VIEW_REFERRER" => "https://example.com",- "action_dispatch.remote_ip" => "1.2.3.4",- ),- ["200", { "Content-Type" => "text/html" }],- 0.2,- )-- events =- DiscourseEvent.track_events(:browser_pageview) do- expect { log_browser_pageview(data) }.to change { BrowserPageviewEvent.count }.by(1)- end-- expect(events).to be_empty- expect(BrowserPageviewEvent.queued_count).to eq(0)-- event = BrowserPageviewEvent.last- expect(event.session_id).to eq(session_id)- expect(event.url).to eq("https://discourse.org")- expect(event.referrer).to eq("https://example.com")- expect(event.country_code).to eq("AU")- expect(event.user_agent).to be_present- expect(event.ip_address.to_s).to eq("1.2.3.4")- expect(BrowserPageviewEvent.sources[event.source]).to eq(- BrowserPageviewEvent::SOURCE_PIGGYBACK,- )- end-- it "skips persisted browser pageviews without a URL" do- session_id = "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx"-- data =- Middleware::RequestTracker.get_data(- env(- "HTTP_DISCOURSE_TRACK_VIEW_DEFERRED" => "1",- "HTTP_DISCOURSE_TRACK_VIEW_SESSION_ID" => session_id,- "action_dispatch.remote_ip" => "1.2.3.4",- ),- ["204", {}],- 0.2,- )-- expect { Middleware::RequestTracker.log_request(data) }.not_to change {- BrowserPageviewEvent.count- }- end-- it "populates normalized_referrer via BrowserPageviewEventUrlNormalizer" do- session_id = "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx"-- data =- Middleware::RequestTracker.get_data(- env(- "HTTP_DISCOURSE_TRACK_VIEW" => "1",- "HTTP_DISCOURSE_TRACK_VIEW_SESSION_ID" => session_id,- "HTTP_DISCOURSE_TRACK_VIEW_URL" => "https://discourse.org",- "HTTP_DISCOURSE_TRACK_VIEW_REFERRER" => "https://www.example.com/path?utm_source=x",- "action_dispatch.remote_ip" => "1.2.3.4",- ),- ["200", { "Content-Type" => "text/html" }],- 0.2,- )-- log_browser_pageview(data)-- event = BrowserPageviewEvent.last- expect(event.referrer).to eq("https://www.example.com/path?utm_source=x")- expect(event.normalized_referrer).to eq("example.com/path")- expect(event.normalized_referrer_version).to eq(- BrowserPageviewEventUrlNormalizer::REFERRER_VERSION,- )- end-- it "stores nil normalized_referrer when the referrer is blank" do- session_id = "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx"-- data =- Middleware::RequestTracker.get_data(- env(- "HTTP_DISCOURSE_TRACK_VIEW" => "1",- "HTTP_DISCOURSE_TRACK_VIEW_SESSION_ID" => session_id,- "HTTP_DISCOURSE_TRACK_VIEW_URL" => "https://discourse.org",- "action_dispatch.remote_ip" => "1.2.3.4",- ),- ["200", { "Content-Type" => "text/html" }],- 0.2,- )-- log_browser_pageview(data)-- event = BrowserPageviewEvent.last- expect(event.normalized_referrer).to be_nil- end-- it "takes precedence even when trigger_browser_pageview_events is also true" do- SiteSetting.trigger_browser_pageview_events = true- session_id = "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx"-- data =- Middleware::RequestTracker.get_data(- env(- "HTTP_DISCOURSE_TRACK_VIEW" => "1",- "HTTP_DISCOURSE_TRACK_VIEW_SESSION_ID" => session_id,- "HTTP_DISCOURSE_TRACK_VIEW_URL" => "https://discourse.org",- "action_dispatch.remote_ip" => "1.2.3.4",- ),- ["200", { "Content-Type" => "text/html" }],- 0.2,- )-- events =- DiscourseEvent.track_events(:browser_pageview) do- expect { log_browser_pageview(data) }.to change { BrowserPageviewEvent.count }.by(1)- end-- expect(events).to be_empty- end-- it "queues browser pageviews while PostgreSQL is readonly" do- session_id = "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx"- Discourse.enable_readonly_mode(Discourse::PG_READONLY_MODE_KEY)-- data =- Middleware::RequestTracker.get_data(- env(- "HTTP_DISCOURSE_TRACK_VIEW" => "1",- "HTTP_DISCOURSE_TRACK_VIEW_SESSION_ID" => session_id,- "HTTP_DISCOURSE_TRACK_VIEW_URL" => "https://discourse.org",- "action_dispatch.remote_ip" => "1.2.3.4",- ),- ["200", { "Content-Type" => "text/html" }],- 0.2,- )+ it "does not persist a BrowserPageviewEvent" do+ expect { log_browser_pageview(pageview_data) }.not_to change { BrowserPageviewEvent.count }+ end+ end+ end- expect { log_browser_pageview(data) }.not_to change { BrowserPageviewEvent.count }- expect(BrowserPageviewEvent.queued_count).to eq(1)+ describe "legacy pageview tracking via /pageview" do+ it "returns 204 without accumulating legacy browser pageviews" do+ initial_count = ApplicationRequest.page_view_anon_browser.sum(:count)+ app_called = false+ middleware =+ Middleware::RequestTracker.new(+ lambda do |env|+ app_called = true+ [200, {}, ["OK"]]+ end,+ )- Discourse.disable_readonly_mode(Discourse::PG_READONLY_MODE_KEY)+ status, =+ middleware.call(+ env(+ :path => "/pageview",+ "REQUEST_METHOD" => "POST",+ "HTTP_DISCOURSE_TRACK_VIEW_DEFERRED" => "true",+ ),+ )+ CachedCounting.flush- expect { flush_browser_pageview_events }.to change { BrowserPageviewEvent.count }.by(1)- expect(BrowserPageviewEvent.queued_count).to eq(0)- end- end+ expect(status).to eq(204)+ expect(app_called).to eq(false)+ expect(ApplicationRequest.page_view_anon_browser.sum(:count)).to eq(initial_count)endenddescribe "beacon pageview tracking via /srv/pv" dobefore do- SiteSetting.dashboard_improvements = true- SiteSetting.trigger_browser_pageview_events = truefreeze_timeApplicationRequest.clear_cache!end@@ -1131,11 +726,10 @@ RSpec.describe Middleware::RequestTracker doexpect(status).to eq(204)end- it "increments beacon-specific counters and fires beacon event with correct data" do- SiteSetting.trigger_browser_pageview_events = true+ it "increments browser counters without emitting beacon events" do+ SiteSetting.dashboard_improvements = falseDiscourseIpInfo.stubs(:get).returns(country_code: "US")middleware = Middleware::RequestTracker.new(lambda { |env| [200, {}, ["OK"]] })- language = "en-US"events =DiscourseEvent.track_events(:beacon_browser_pageview) do@@ -1144,7 +738,6 @@ RSpec.describe Middleware::RequestTracker do{url: "https://test.com/t/topic/123",referrer: "https://test.com/",- language:,session_id: "abc123",topic_id: 123,},@@ -1158,21 +751,10 @@ RSpec.describe Middleware::RequestTracker doexpect(ApplicationRequest.page_view_anon_browser_beacon.first.count).to eq(1)expect(ApplicationRequest.page_view_anon.first).to be_nilexpect(ApplicationRequest.page_view_anon_browser.first).to be_nil-- event = events[0][:params].last- expect(event[:url]).to eq("https://test.com/t/topic/123")- expect(event[:referrer]).to eq("https://test.com/")- expect(event[:language]).to eq(language)- expect(event[:session_id]).to eq("abc123")- expect(event[:topic_id]).to eq(123)- expect(event[:country_code]).to eq("US")- expect(event[:user_agent]).to eq(- "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",- )+ expect(events).to be_emptyend- it "persists beacon pageviews to browser_pageview_events with beacon source" do- SiteSetting.persist_browser_pageview_events = true+ it "persists beacon pageviews to browser_pageview_events" doDiscourseIpInfo.stubs(:get).returns(country_code: "US")middleware = Middleware::RequestTracker.new(lambda { |env| [200, {}, ["OK"]] })@@ -1199,17 +781,15 @@ RSpec.describe Middleware::RequestTracker doexpect(event.topic_id).to eq(123)expect(event.country_code).to eq("US")expect(event.ip_address.to_s).to eq("1.2.3.4")- expect(event.source).to eq("beacon")end- it "increments legacy and BPV counters from non-beacon requests" do+ it "increments legacy counters from non-beacon requests" domiddleware = Middleware::RequestTracker.new(lambda { |env| [200, {}, ["OK"]] })middleware.call(env("HTTP_DISCOURSE_TRACK_VIEW" => "1"))CachedCounting.flushexpect(ApplicationRequest.page_view_anon.first.count).to eq(1)expect(ApplicationRequest.page_view_anon_browser.first.count).to eq(1)-expect(ApplicationRequest.page_view_anon_browser_beacon.first).to be_nilend@@ -1270,45 +850,21 @@ RSpec.describe Middleware::RequestTracker doexpect(status).to eq(204)end- context "when dashboard_improvements is disabled" do- before { SiteSetting.dashboard_improvements = false }-- it "returns the app's response for beacon requests instead of 204" do- app_called = false- middleware =- Middleware::RequestTracker.new(- lambda do |env|- app_called = true- [404, {}, ["unknown app path"]]- end,- )- status, = middleware.call(beacon_env({}))-- expect(status).to eq(404)- expect(app_called).to eq(true)- end- end-- context "when browser pageview persistence and events are disabled" do- before do- SiteSetting.persist_browser_pageview_events = false- SiteSetting.trigger_browser_pageview_events = false- end+ it "accepts beacon requests when dashboard improvements are disabled" do+ SiteSetting.dashboard_improvements = false+ app_called = false+ middleware =+ Middleware::RequestTracker.new(+ lambda do |env|+ app_called = true+ [404, {}, ["unknown app path"]]+ end,+ )- it "returns the app's response for beacon requests instead of 204" do- app_called = false- middleware =- Middleware::RequestTracker.new(- lambda do |env|- app_called = true- [404, {}, ["unknown app path"]]- end,- )- status, = middleware.call(beacon_env({}))+ status, = middleware.call(beacon_env({}, same_origin))- expect(status).to eq(404)- expect(app_called).to eq(true)- end+ expect(status).to eq(204)+ expect(app_called).to eq(false)endend@@ -2018,8 +1574,6 @@ RSpec.describe Middleware::RequestTracker doenddescribe "session engagement tracking via /srv/se" do- before { SiteSetting.persist_browser_pageview_events = true }-def engagement_env(body_hash, extra = {})env({@@ -2172,17 +1726,5 @@ RSpec.describe Middleware::RequestTracker doexpect(status).to eq(204)}.not_to change { BrowserPageviewSessionEngagement.count }end-- it "does not intercept the request when persist_browser_pageview_events is disabled" do- SiteSetting.persist_browser_pageview_events = false- SiteSetting.trigger_browser_pageview_events = true- SiteSetting.dashboard_improvements = true- middleware = Middleware::RequestTracker.new(lambda { |_env| [200, {}, ["OK"]] })-- expect {- status, = middleware.call(engagement_env(payload, same_origin))- expect(status).to eq(200)- }.not_to change { BrowserPageviewSessionEngagement.count }- endendenddiff --git a/spec/lib/statistics_spec.rb b/spec/lib/statistics_spec.rbindex 1ddf493036c..aaea0ab4843 100644--- a/spec/lib/statistics_spec.rb+++ b/spec/lib/statistics_spec.rb@@ -212,6 +212,23 @@ RSpec.describe Statistics doafter { ApplicationRequest.disable }+ it "preserves piggyback visitor estimates on the first partial beacon day" do+ ApplicationRequest.create!(+ date: date.to_date,+ req_type: :page_view_anon_browser_beacon,+ count: 8,+ )+ ApplicationRequest.create!(+ date: date.to_date,+ req_type: :page_view_logged_in_browser_beacon,+ count: 2,+ )++ freeze_time(date) do+ expect(described_class.visitors).to eq(last_day: 3, "7_days": 6, "30_days": 9)+ end+ end+it "estimates the number of visitors for each of the previous 1 day, 7 days and 30 days periods" dofreeze_time(date) dovisitors = described_class.visitors@@ -284,6 +301,23 @@ RSpec.describe Statistics doafter { ApplicationRequest.disable }+ it "preserves piggyback EU visitor estimates on the first partial beacon day" do+ ApplicationRequest.create!(+ date: date.to_date,+ req_type: :page_view_anon_browser_beacon,+ count: 8,+ )+ ApplicationRequest.create!(+ date: date.to_date,+ req_type: :page_view_logged_in_browser_beacon,+ count: 2,+ )++ freeze_time(date) do+ expect(described_class.eu_visitors).to eq(last_day: 3, "7_days": 3, "30_days": 5)+ end+ end+it "estimates the number of EU visitors for each of the previous 1 day, 7 days and 30 days periods" dofreeze_time(date) doeu_visitors = described_class.eu_visitorsdiff --git a/spec/migrations/archive_duplicated_legacy_browser_pageview_events_spec.rb b/spec/migrations/archive_duplicated_legacy_browser_pageview_events_spec.rbnew file mode 100644index 00000000000..fe0db4ae4e0--- /dev/null+++ b/spec/migrations/archive_duplicated_legacy_browser_pageview_events_spec.rb@@ -0,0 +1,75 @@+# frozen_string_literal: true++require Rails.root.join(+ "db/migrate/20260910030404_archive_duplicated_legacy_browser_pageview_events.rb",+ )++RSpec.describe ArchiveDuplicatedLegacyBrowserPageviewEvents do+ before do+ @original_verbose = ActiveRecord::Migration.verbose+ ActiveRecord::Migration.verbose = false+ DB.exec(+ "ALTER TABLE browser_pageview_events DISABLE TRIGGER skip_piggyback_browser_pageview_events",+ )+ end++ after do+ DB.exec(+ "ALTER TABLE browser_pageview_events ENABLE TRIGGER skip_piggyback_browser_pageview_events",+ )+ ActiveRecord::Migration.verbose = @original_verbose+ end++ it "archives duplicated legacy events after batches without matches" do+ standalone_legacy = Fabricate(:browser_pageview_event, session_id: "legacy-session")+ another_standalone_legacy =+ Fabricate(:browser_pageview_event, session_id: "another-legacy-session")+ duplicated_legacy = Fabricate(:browser_pageview_event, session_id: "duplicated-session")+ beacon = Fabricate(:browser_pageview_event, session_id: "duplicated-session")+ standalone_beacon = Fabricate(:browser_pageview_event, session_id: "beacon-session")+ duplicated_score = BrowserPageviewEventScore.create!(event_id: duplicated_legacy.id)+ standalone_legacy_score = BrowserPageviewEventScore.create!(event_id: standalone_legacy.id)+ beacon_score = BrowserPageviewEventScore.create!(event_id: beacon.id)+ DB.exec(+ "UPDATE browser_pageview_events SET source = 1 WHERE id IN (:ids)",+ ids: [standalone_legacy.id, another_standalone_legacy.id, duplicated_legacy.id],+ )+ DB.exec(+ "ALTER TABLE browser_pageview_events ENABLE TRIGGER skip_piggyback_browser_pageview_events",+ )++ event_data =+ DB.query_single(+ "SELECT to_jsonb(events) FROM browser_pageview_events events WHERE id = :id",+ id: duplicated_legacy.id,+ )+ score_data =+ DB.query_single(+ "SELECT to_jsonb(scores) FROM browser_pageview_event_scores scores WHERE id = :id",+ id: duplicated_score.id,+ )++ stub_const(described_class, "BATCH_SIZE", 2) { described_class.new.up }+ described_class.new.up++ expect(+ DB.query_single("SELECT to_jsonb(events) FROM browser_pageview_events_backup events"),+ ).to eq(event_data)+ expect(+ DB.query_single("SELECT to_jsonb(scores) FROM browser_pageview_event_scores_backup scores"),+ ).to eq(score_data)++ expect(BrowserPageviewEvent.pluck(:id)).to contain_exactly(+ beacon.id,+ standalone_legacy.id,+ another_standalone_legacy.id,+ standalone_beacon.id,+ )+ expect(BrowserPageviewEvent.where(id: duplicated_legacy.id)).not_to exist+ expect(BrowserPageviewEventScore.pluck(:id)).to contain_exactly(+ standalone_legacy_score.id,+ beacon_score.id,+ )+ expect(BrowserPageviewEventScore.where(id: duplicated_score.id)).not_to exist+ end+enddiff --git a/spec/migrations/archive_piggyback_browser_pageviews_after_beacon_cutover_spec.rb b/spec/migrations/archive_piggyback_browser_pageviews_after_beacon_cutover_spec.rbnew file mode 100644index 00000000000..b1b840e7e1e--- /dev/null+++ b/spec/migrations/archive_piggyback_browser_pageviews_after_beacon_cutover_spec.rb@@ -0,0 +1,123 @@+# frozen_string_literal: true++require Rails.root.join(+ "db/migrate/20260910030427_archive_piggyback_browser_pageviews_after_beacon_cutover.rb",+ )++RSpec.describe ArchivePiggybackBrowserPageviewsAfterBeaconCutover do+ before do+ DB.exec(+ "ALTER TABLE browser_pageview_events DISABLE TRIGGER skip_piggyback_browser_pageview_events",+ )+ end++ after do+ DB.exec(+ "ALTER TABLE browser_pageview_events ENABLE TRIGGER skip_piggyback_browser_pageview_events",+ )+ end++ def legacy_event(created_at)+ event = Fabricate(:browser_pageview_event, created_at:)+ DB.exec("UPDATE browser_pageview_events SET source = 1 WHERE id = :id", id: event.id)+ event+ end++ it "archives unmatched piggybacks from midnight after the first partial beacon day" do+ first_day = Date.new(2026, 8, 20)+ ApplicationRequest.create!(date: first_day, req_type: :page_view_anon_browser_beacon, count: 10)+ ApplicationRequest.create!(date: first_day, req_type: :page_view_anon_browser, count: 20)+ historical = legacy_event(Time.utc(2026, 8, 19))+ partial_day = legacy_event(Time.utc(2026, 8, 20, 23, 59, 59))+ boundary = legacy_event(Time.utc(2026, 8, 21))+ later = legacy_event(Time.utc(2026, 8, 22))+ beacon = Fabricate(:browser_pageview_event, created_at: later.created_at)+ score = BrowserPageviewEventScore.create!(event_id: boundary.id, velocity_score: 42)+ historical_score = BrowserPageviewEventScore.create!(event_id: historical.id)+ event_data =+ DB.query_single(+ "SELECT to_jsonb(events) FROM browser_pageview_events events WHERE id IN (:ids) ORDER BY id",+ ids: [boundary.id, later.id],+ )+ score_data =+ DB.query_single(+ "SELECT to_jsonb(scores) FROM browser_pageview_event_scores scores WHERE id = :id",+ id: score.id,+ )++ stub_const(described_class, "BATCH_SIZE", 1) { described_class.new.up }+ described_class.new.up++ expect(BrowserPageviewEvent.pluck(:id)).to contain_exactly(+ historical.id,+ partial_day.id,+ beacon.id,+ )+ expect(BrowserPageviewEventScore.pluck(:id)).to contain_exactly(historical_score.id)+ expect(+ DB.query_single(+ "SELECT to_jsonb(events) FROM browser_pageview_events_backup events ORDER BY id",+ ),+ ).to eq(event_data)+ expect(+ DB.query_single("SELECT to_jsonb(scores) FROM browser_pageview_event_scores_backup scores"),+ ).to eq(score_data)+ end++ it "uses the first positive beacon day when legacy counters on that day are zero" do+ first_day = Date.new(2026, 8, 20)+ ApplicationRequest.create!(+ date: first_day - 1,+ req_type: :page_view_anon_browser_beacon,+ count: 0,+ )+ ApplicationRequest.create!(+ date: first_day,+ req_type: :page_view_logged_in_browser_beacon,+ count: 1,+ )+ ApplicationRequest.create!(date: first_day, req_type: :page_view_logged_in_browser, count: 0)+ historical = legacy_event(Time.utc(2026, 8, 19, 23, 59, 59))+ boundary = legacy_event(Time.utc(2026, 8, 20))++ described_class.new.up++ expect(BrowserPageviewEvent.pluck(:id)).to eq([historical.id])+ expect(DB.query_single("SELECT id FROM browser_pageview_events_backup")).to eq([boundary.id])+ end++ it "preserves all piggybacks when no positive beacon counters exist" do+ ApplicationRequest.create!(+ date: Date.new(2026, 8, 20),+ req_type: :page_view_anon_browser_beacon,+ count: 0,+ )+ legacy = legacy_event(Time.utc(2026, 8, 21))++ described_class.new.up++ expect(BrowserPageviewEvent.pluck(:id)).to eq([legacy.id])+ expect(DB.query_single("SELECT id FROM browser_pageview_events_backup")).to be_empty+ end++ it "rolls back the whole batch if a score cannot be backed up" do+ ApplicationRequest.create!(+ date: Date.new(2026, 8, 20),+ req_type: :page_view_anon_browser_beacon,+ count: 1,+ )+ legacy = legacy_event(Time.utc(2026, 8, 21))+ score = BrowserPageviewEventScore.create!(event_id: legacy.id)+ DB.exec(+ "INSERT INTO browser_pageview_event_scores_backup SELECT * FROM browser_pageview_event_scores",+ )++ expect do+ BrowserPageviewEvent.transaction(requires_new: true) { described_class.new.up }+ end.to raise_error(PG::UniqueViolation)++ expect(BrowserPageviewEvent.pluck(:id)).to eq([legacy.id])+ expect(BrowserPageviewEventScore.pluck(:id)).to eq([score.id])+ expect(DB.query_single("SELECT id FROM browser_pageview_events_backup")).to be_empty+ end+enddiff --git a/spec/migrations/remove_browser_pageview_event_site_settings_spec.rb b/spec/migrations/remove_browser_pageview_event_site_settings_spec.rbnew file mode 100644index 00000000000..30ca6bc5ddc--- /dev/null+++ b/spec/migrations/remove_browser_pageview_event_site_settings_spec.rb@@ -0,0 +1,26 @@+# frozen_string_literal: true++require Rails.root.join("db/migrate/20260902075239_remove_browser_pageview_event_site_settings.rb")++RSpec.describe RemoveBrowserPageviewEventSiteSettings do+ before do+ @original_verbose = ActiveRecord::Migration.verbose+ ActiveRecord::Migration.verbose = false+ end++ after { ActiveRecord::Migration.verbose = @original_verbose }++ it "removes the obsolete settings and preserves unrelated settings" do+ data_type = SiteSettings::TypeSupervisor.types[:bool]+ SiteSetting.create!(name: "persist_browser_pageview_events", value: "f", data_type:)+ SiteSetting.create!(name: "trigger_browser_pageview_events", value: "t", data_type:)+ unrelated_setting = SiteSetting.create!(name: "unrelated_setting", value: "t", data_type:)++ described_class.new.up++ expect(+ SiteSetting.where(name: %w[persist_browser_pageview_events trigger_browser_pageview_events]),+ ).to be_empty+ expect(unrelated_setting.reload.value).to eq("t")+ end+enddiff --git a/spec/migrations/set_browser_pageview_event_source_default_to_beacon_spec.rb b/spec/migrations/set_browser_pageview_event_source_default_to_beacon_spec.rbnew file mode 100644index 00000000000..3bf12415624--- /dev/null+++ b/spec/migrations/set_browser_pageview_event_source_default_to_beacon_spec.rb@@ -0,0 +1,80 @@+# frozen_string_literal: true++require Rails.root.join(+ "db/migrate/20260831011840_set_browser_pageview_event_source_default_to_beacon.rb",+ )++require Rails.root.join(+ "db/migrate/20260910030404_archive_duplicated_legacy_browser_pageview_events.rb",+ )++RSpec.describe SetBrowserPageviewEventSourceDefaultToBeacon do+ %w[default explicit].each do |beacon_source|+ it "preserves legacy events when a #{beacon_source} beacon arrives after archiving" do+ migration = described_class.new+ migration.suppress_messages { migration.down }++ legacy = Fabricate(:browser_pageview_event, session_id: "shared-session")+ another_legacy = Fabricate(:browser_pageview_event, session_id: legacy.session_id)+ unrelated_legacy = Fabricate(:browser_pageview_event)+ legacy_score = BrowserPageviewEventScore.create!(event_id: legacy.id)+ another_legacy_score = BrowserPageviewEventScore.create!(event_id: another_legacy.id)+ unrelated_score = BrowserPageviewEventScore.create!(event_id: unrelated_legacy.id)++ migration.suppress_messages do+ migration.up+ ArchiveDuplicatedLegacyBrowserPageviewEvents.new.up+ end+ expect(BrowserPageviewEvent.count).to eq(3)++ source_column = beacon_source == "explicit" ? ", source" : ""+ source_value = beacon_source == "explicit" ? ", 2" : ""+ beacon_ids = DB.query_single(<<~SQL, session_id: legacy.session_id)+ INSERT INTO browser_pageview_events+ (url, ip_address, user_agent, session_id, created_at#{source_column})+ VALUES+ ('/first', '1.2.3.4', 'test', :session_id, CURRENT_TIMESTAMP#{source_value}),+ ('/second', '1.2.3.4', 'test', :session_id, CURRENT_TIMESTAMP#{source_value})+ RETURNING id+ SQL++ expect(BrowserPageviewEvent.pluck(:id)).to contain_exactly(+ legacy.id,+ another_legacy.id,+ unrelated_legacy.id,+ *beacon_ids,+ )+ expect(BrowserPageviewEventScore.pluck(:id)).to contain_exactly(+ legacy_score.id,+ another_legacy_score.id,+ unrelated_score.id,+ )+ end+ end++ it "skips piggyback writes while allowing explicit and default beacon sources" do+ migration = described_class.new+ migration.suppress_messages do+ migration.down+ migration.up+ end+ beacon = Fabricate(:browser_pageview_event)++ DB.exec(<<~SQL, id: beacon.id)+ INSERT INTO browser_pageview_events+ (url, ip_address, user_agent, session_id, created_at, source)+ SELECT url, ip_address, user_agent, session_id, created_at, sources.source+ FROM browser_pageview_events+ CROSS JOIN (VALUES (1), (2)) AS sources(source)+ WHERE id = :id+ SQL++ expect(DB.query_single("SELECT source FROM browser_pageview_events")).to contain_exactly(2, 2)++ DB.exec("UPDATE browser_pageview_events SET source = 1 WHERE id = :id", id: beacon.id)++ expect(+ DB.query_single("SELECT source FROM browser_pageview_events WHERE id = :id", id: beacon.id),+ ).to eq([2])+ end+enddiff --git a/spec/models/application_request_spec.rb b/spec/models/application_request_spec.rbindex 0fa324e8f6c..9c3bf1b02c7 100644--- a/spec/models/application_request_spec.rb+++ b/spec/models/application_request_spec.rb@@ -16,6 +16,49 @@ RSpec.describe ApplicationRequest doApplicationRequest.increment!(key)end+ describe ".browser_pageviews" do+ it "preserves the first partial beacon day and switches both cohorts the following day" do+ today = Date.current+ history =+ described_class.create!(date: today - 2, req_type: :page_view_anon_browser, count: 4)+ described_class.create!(date: today - 2, req_type: :page_view_anon_browser_beacon, count: 0)+ first_day =+ described_class.create!(date: today - 1, req_type: :page_view_anon_browser, count: 1000)+ described_class.create!(date: today - 1, req_type: :page_view_anon_browser_beacon, count: 50)+ described_class.create!(+ date: today - 1,+ req_type: :page_view_logged_in_browser_beacon,+ count: 5,+ )+ beacon =+ described_class.create!(date: today, req_type: :page_view_anon_browser_beacon, count: 8)+ described_class.create!(date: today, req_type: :page_view_anon_browser, count: 3)+ described_class.create!(+ date: today,+ req_type: :page_view_anon_browser_mobile_beacon,+ count: 2,+ )++ expect(described_class.browser_pageviews).to contain_exactly(history, first_day, beacon)+ end++ it "includes the first beacon day when piggyback counters are absent or zero" do+ today = Date.current+ described_class.create!(date: today, req_type: :page_view_anon_browser, count: 0)+ beacon =+ described_class.create!(date: today, req_type: :page_view_anon_browser_beacon, count: 8)++ expect(described_class.browser_pageviews).to contain_exactly(beacon)+ end++ it "uses piggyback history when no beacons have been recorded" do+ history =+ described_class.create!(date: Date.current, req_type: :page_view_anon_browser, count: 4)++ expect(described_class.browser_pageviews).to contain_exactly(history)+ end+ end+it "can log app requests" dofreeze_timed1 = Time.now.utc.to_datediff --git a/spec/models/browser_pageview_event_spec.rb b/spec/models/browser_pageview_event_spec.rbindex 7e0d07ee0d8..6f96f27e7c9 100644--- a/spec/models/browser_pageview_event_spec.rb+++ b/spec/models/browser_pageview_event_spec.rb@@ -11,177 +11,6 @@ RSpec.describe BrowserPageviewEvent doDiscourse.clear_readonly!end- describe ".beacon_cutover_date" do- it "returns the day after dashboard improvements was manually opted into" do- SiteSetting.dashboard_improvements = true- UpcomingChangeEvent.create!(- upcoming_change_name: "dashboard_improvements",- event_type: :manual_opt_in,- created_at: Time.zone.local(2026, 5, 1, 10),- )-- expect(described_class.beacon_cutover_date).to eq(Date.new(2026, 5, 2))- end-- it "returns the day after dashboard improvements was automatically promoted" do- SiteSetting.dashboard_improvements = true- UpcomingChangeEvent.create!(- upcoming_change_name: "dashboard_improvements",- event_type: :automatically_promoted,- created_at: Time.zone.local(2026, 5, 1, 10),- )-- expect(described_class.beacon_cutover_date).to eq(Date.new(2026, 5, 2))- end-- it "falls back to the site setting row when no enablement event exists" do- SiteSetting.dashboard_improvements = true- SiteSetting- .stubs(:where)- .with(name: "dashboard_improvements")- .returns(stub(maximum: Time.zone.local(2026, 5, 1, 10)))-- expect(described_class.beacon_cutover_date).to eq(Date.new(2026, 5, 2))- end-- it "uses the most recent enablement time across events and the setting row" do- SiteSetting.dashboard_improvements = true- UpcomingChangeEvent.create!(- upcoming_change_name: "dashboard_improvements",- event_type: :automatically_promoted,- created_at: Time.zone.local(2026, 5, 1, 10),- )- SiteSetting- .stubs(:where)- .with(name: "dashboard_improvements")- .returns(stub(maximum: Time.zone.local(2026, 5, 5, 10)))-- expect(described_class.beacon_cutover_date).to eq(Date.new(2026, 5, 6))- end-- it "returns nil when dashboard improvements are disabled" do- SiteSetting.dashboard_improvements = false-- expect(described_class.beacon_cutover_date).to be_nil- end-- it "falls back to the first beacon counter date when no enablement record exists" do- SiteSetting.dashboard_improvements = true- Fabricate(:anonymous_browser_beacon_application_request, date: "2026-05-04", count: 2)- Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-05", count: 1)-- expect(described_class.beacon_cutover_date).to eq(Date.new(2026, 5, 4))- end-- it "returns nil when no beacons are collected" do- SiteSetting.dashboard_improvements = true- SiteSetting.trigger_browser_pageview_events = false- SiteSetting.persist_browser_pageview_events = false- UpcomingChangeEvent.create!(- upcoming_change_name: "dashboard_improvements",- event_type: :manual_opt_in,- )-- expect(described_class.beacon_cutover_date).to be_nil- end-- it "returns nil when legacy pageviews are enabled" do- SiteSetting.use_legacy_pageviews = true- SiteSetting.dashboard_improvements = true-- expect(described_class.beacon_cutover_date).to be_nil- end- end-- describe ".rollup_source_condition" do- it "matches only piggyback events when there is no cutover date" do- SiteSetting.dashboard_improvements = false- piggyback = Fabricate(:browser_pageview_event, source: :piggyback)- Fabricate(:browser_pageview_event, source: :beacon)-- bounded_condition =- described_class.rollup_source_condition(- start_date: Date.new(2026, 6, 1),- end_date: Date.new(2026, 6, 20),- )-- expect(described_class.rollup_source_condition).to eq(- "source = #{described_class::SOURCE_PIGGYBACK}",- )- expect(bounded_condition).to eq("source = #{described_class::SOURCE_PIGGYBACK}")- expect(described_class.where(described_class.rollup_source_condition)).to contain_exactly(- piggyback,- )- end-- it "matches piggyback events before the cutover date and beacon events from it onwards" do- SiteSetting.dashboard_improvements = true- UpcomingChangeEvent.create!(- upcoming_change_name: "dashboard_improvements",- event_type: :manual_opt_in,- created_at: Time.utc(2026, 6, 10, 9),- )-- pre_piggyback =- Fabricate(:browser_pageview_event, source: :piggyback, created_at: Time.utc(2026, 6, 9))- Fabricate(:browser_pageview_event, source: :beacon, created_at: Time.utc(2026, 6, 9))- Fabricate(:browser_pageview_event, source: :piggyback, created_at: Time.utc(2026, 6, 15))- post_beacon =- Fabricate(:browser_pageview_event, source: :beacon, created_at: Time.utc(2026, 6, 15))-- expect(described_class.where(described_class.rollup_source_condition)).to contain_exactly(- pre_piggyback,- post_beacon,- )- end-- it "qualifies columns with the given table alias" do- SiteSetting.dashboard_improvements = false- condition = described_class.rollup_source_condition(table: "e")-- expect(condition).to eq("e.source = #{described_class::SOURCE_PIGGYBACK}")- end-- it "matches only piggyback events when the bounded range ends at the cutover" do- described_class.stubs(:beacon_cutover_date).returns(Date.new(2026, 6, 10))-- condition =- described_class.rollup_source_condition(- start_date: Date.new(2026, 6, 1),- end_date: Date.new(2026, 6, 10),- )-- expect(condition).to eq("source = #{described_class::SOURCE_PIGGYBACK}")- end-- it "matches only beacon events when the bounded range starts at the cutover" do- described_class.stubs(:beacon_cutover_date).returns(Date.new(2026, 6, 10))-- condition =- described_class.rollup_source_condition(- start_date: Date.new(2026, 6, 10),- end_date: Date.new(2026, 6, 20),- )-- expect(condition).to eq("source = #{described_class::SOURCE_BEACON}")- end-- it "keeps the mixed source condition when the bounded range crosses the cutover" do- described_class.stubs(:beacon_cutover_date).returns(Date.new(2026, 6, 10))-- condition =- described_class.rollup_source_condition(- start_date: Date.new(2026, 6, 1),- end_date: Date.new(2026, 6, 20),- )-- expect(condition).to include(- "created_at < '2026-06-10' AND source = #{described_class::SOURCE_PIGGYBACK}",- "created_at >= '2026-06-10' AND source = #{described_class::SOURCE_BEACON}",- )- end- end-it "truncates string fields before saving" doevent =described_class.create!(@@ -218,7 +47,6 @@ RSpec.describe BrowserPageviewEvent dolanguage: "en-AU",session_id: "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx",topic_id: 123,- source: described_class::SOURCE_BEACON,occurred_at: occurred_at.iso8601(6),}end@@ -248,13 +76,27 @@ RSpec.describe BrowserPageviewEvent do)expect(event.normalized_referrer).to eq("example.com/path")expect(event.created_at).to eq_time(occurred_at)- expect(event.source).to eq("beacon")expect(event.browser).to eq("edge")expect(event.language).to eq("en-AU")expect(event.normalized_language).to eq("en")expect(described_class.queued_count).to eq(0)end+ it "discards queued piggyback events while preserving old and new beacon payloads" do+ [1, 2].each do |source|+ Discourse.redis.rpush(+ described_class::REDIS_QUEUE_KEY,+ JSON.generate(payload.merge(source: source, url: "/source-#{source}")),+ )+ end+ queue_payload(payload)++ expect(described_class.flush_queued!).to eq(3)++ expect(described_class.pluck(:url)).to contain_exactly("/source-2", payload[:url])+ expect(described_class.queued_count).to eq(0)+ end+it "keeps queued payloads while PostgreSQL is readonly" doDiscourse.stubs(:pg_readonly_mode?).returns(true)described_class.enqueue_for_later(payload)diff --git a/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb b/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rbindex 3b7572c26e5..3acda4d6721 100644--- a/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb+++ b/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb@@ -161,28 +161,6 @@ RSpec.describe BrowserPageviewSessionEngagementDailyRollup doexpect(described_class.count).to eq(0)end- it "counts only rollup-source pageviews, even within a single session" do- SiteSetting.dashboard_improvements = true- UpcomingChangeEvent.create!(- upcoming_change_name: "dashboard_improvements",- event_type: :manual_opt_in,- created_at: Time.utc(2026, 6, 1, 9),- )- beacon_event =- Fabricate(:browser_pageview_event, source: :beacon, created_at: Time.utc(2026, 6, 10, 9))- Fabricate(- :browser_pageview_event,- session_id: beacon_event.session_id,- source: :piggyback,- created_at: Time.utc(2026, 6, 10, 10),- )- Fabricate(:browser_pageview_event, source: :piggyback, created_at: Time.utc(2026, 6, 10, 9))-- described_class.aggregate(start_date:, end_date:)-- expect(described_class.all).to contain_exactly(have_attributes(sessions: 1, bounced: 1))- end-it "clears a session's previous logged-in partition when it flips across runs" doevent = Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 10, 9))described_class.aggregate(start_date:, end_date:)diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rbindex b7d2f5d347a..788db5c9c23 100644--- a/spec/models/report_spec.rb+++ b/spec/models/report_spec.rb@@ -406,6 +406,24 @@ RSpec.describe Report dolet(:report) { Report.find("page_view_total_reqs") }+ it "combines historical piggyback totals with beacon totals without counting overlap" do+ SiteSetting.use_legacy_pageviews = false+ ApplicationRequest.create!(+ date: Date.current - 2,+ req_type: :page_view_anon_browser,+ count: 4,+ )+ ApplicationRequest.create!(date: Date.current, req_type: :page_view_anon_browser, count: 3)+ ApplicationRequest.create!(+ date: Date.current,+ req_type: :page_view_anon_browser_beacon,+ count: 8,+ )++ expect(report.data).to eq([{ x: Date.current - 2, y: 4 }, { x: Date.current, y: 3 }])+ expect(report.total).to eq(7)+ end+context "with no data" doit "returns no page-view requests" doexpect(report.data).to be_empty@@ -1791,6 +1809,58 @@ RSpec.describe Report dolet(:reports) { Report.find("site_traffic") }+ it "reports initial beacon pageviews even without piggyback history" do+ ApplicationRequest.create!(+ date: Date.current,+ req_type: :page_view_anon_browser_beacon,+ count: 8,+ )+ ApplicationRequest.create!(+ date: Date.current,+ req_type: :page_view_logged_in_browser_beacon,+ count: 2,+ )+ ApplicationRequest.create!(date: Date.current, req_type: :page_view_anon, count: 9)+ ApplicationRequest.create!(date: Date.current, req_type: :page_view_logged_in, count: 3)++ series = reports.data.to_h { |entry| [entry[:req], entry[:data]] }++ expect(series["page_view_anon_browser"]).to eq([{ x: Date.current, y: 8 }])+ expect(series["page_view_logged_in_browser"]).to eq([{ x: Date.current, y: 2 }])+ expect(series["page_view_other"]).to eq([{ x: Date.current, y: 2 }])+ end++ it "uses historical piggyback counts before switching both browser series to beacons" do+ ApplicationRequest.create!(+ date: Date.current - 2,+ req_type: :page_view_anon_browser,+ count: 4,+ )+ ApplicationRequest.create!(date: Date.current, req_type: :page_view_anon_browser, count: 3)+ ApplicationRequest.create!(+ date: Date.current,+ req_type: :page_view_logged_in_browser,+ count: 6,+ )+ ApplicationRequest.create!(+ date: Date.current,+ req_type: :page_view_anon_browser_beacon,+ count: 8,+ )++ series = reports.data.to_h { |entry| [entry[:req], entry[:data]] }++ expect(series["page_view_anon_browser"]).to eq(+ [{ x: Date.current - 2, y: 4 }, { x: Date.current, y: 3 }],+ )+ expect(series["page_view_logged_in_browser"]).to eq(+ [{ x: Date.current - 2, y: 0 }, { x: Date.current, y: 6 }],+ )+ expect(series["page_view_other"]).to eq(+ [{ x: Date.current - 2, y: 0 }, { x: Date.current, y: 0 }],+ )+ end+context "with no data" doit "returns empty site-traffic series" doreports.data.each { |report| expect(report[:data]).to be_empty }@@ -2471,31 +2541,5 @@ RSpec.describe Report doendendend-- context "with browser pageview reports" do- it "hides them from admins when persist_browser_pageview_events is disabled" do- SiteSetting.persist_browser_pageview_events = false-- Report::BROWSER_PAGEVIEW_REPORTS.each do |report_type|- expect(Report.hidden?(report_type, guardian: admin_guardian)).to eq(true)- end- end-- it "exposes them to admins when persist_browser_pageview_events is enabled" do- SiteSetting.persist_browser_pageview_events = true-- Report::BROWSER_PAGEVIEW_REPORTS.each do |report_type|- expect(Report.hidden?(report_type, guardian: admin_guardian)).to eq(false)- end- end-- it "always hides them from moderators, even when persist_browser_pageview_events is enabled" do- SiteSetting.persist_browser_pageview_events = true-- Report::BROWSER_PAGEVIEW_REPORTS.each do |report_type|- expect(Report.hidden?(report_type, guardian: moderator_guardian)).to eq(true)- end- end- endendenddiff --git a/spec/requests/admin/dashboard_controller_spec.rb b/spec/requests/admin/dashboard_controller_spec.rbindex 28efbf73ea1..e3bae06dd17 100644--- a/spec/requests/admin/dashboard_controller_spec.rb+++ b/spec/requests/admin/dashboard_controller_spec.rb@@ -187,8 +187,6 @@ RSpec.describe Admin::DashboardController doendcontext "with traffic_data" do- before { SiteSetting.persist_browser_pageview_events = false }-let(:traffic_data) { section_payloads["traffic"]&.dig("data") }it "returns the site traffic payload for the selected dates" do@@ -220,6 +218,12 @@ RSpec.describe Admin::DashboardController do"logged_in_share" => {"value" => 33,},+ "bounce_rate" => {+ "value" => nil,+ },+ "average_session_duration_seconds" => {+ "value" => nil,+ },},"pageview_series" => [{@@ -263,12 +267,23 @@ RSpec.describe Admin::DashboardController do],},],+ "top_countries" => {+ "rows" => [],+ "error" => nil,+ },+ "top_referrers" => {+ "rows" => [],+ "error" => nil,+ },+ "top_entry_urls" => {+ "rows" => [],+ "error" => nil,+ },)endit "does not expose admin-only browser pageview cards to moderators" doSiteSetting.use_legacy_pageviews = false- SiteSetting.persist_browser_pageview_events = trueconfigure_dashboard_sections(%w[traffic])country_code = "US"@@ -287,7 +302,6 @@ RSpec.describe Admin::DashboardController docountry_code: country_code,normalized_referrer: normalized_referrer,created_at: event_date,- source: "beacon",)end@@ -1422,9 +1436,7 @@ RSpec.describe Admin::DashboardController dofreeze_time(Time.zone.local(2026, 5, 14, 12, 0, 0))SiteSetting.dashboard_improvements = trueSiteSetting.improved_crawler_detection = true- SiteSetting.persist_browser_pageview_events = trueSiteSetting.use_legacy_pageviews = false- BrowserPageviewEvent.stubs(:beacon_cutover_date).returns(Date.new(2026, 1, 1))Discourse.stubs(:current_hostname).returns("test.localhost")DiscourseIpInfo.stubs(:get).returns(asn: 64_496, organization: "Example Network")DiscourseIpInfo@@ -1460,7 +1472,6 @@ RSpec.describe Admin::DashboardController dosession_id: "admin-session",normalized_referrer: "search.example/results?q=discourse",normalized_referrer_version: BrowserPageviewEventUrlNormalizer::REFERRER_VERSION,- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: "2026-05-10 10:00:00",)end@@ -1477,7 +1488,6 @@ RSpec.describe Admin::DashboardController dosession_id: "admin-session",normalized_referrer: "test.localhost/landing",normalized_referrer_version: BrowserPageviewEventUrlNormalizer::REFERRER_VERSION,- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: "2026-05-10 10:01:00",)end@@ -1491,7 +1501,6 @@ RSpec.describe Admin::DashboardController doip_address: "198.51.100.2",user_agent: "Mozilla/5.0 Firefox/126.0",session_id: "anonymous-session",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: "2026-05-11 10:00:00",)end@@ -1669,7 +1678,6 @@ RSpec.describe Admin::DashboardController dourl: "https://test.localhost/same-site-full-load",normalized_referrer: "test.localhost/previous-page",session_id: "same-site-full-load",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: "2026-05-11 11:00:00",)@@ -1709,7 +1717,6 @@ RSpec.describe Admin::DashboardController doip_address: "192.0.2.1",user_agent: chrome,session_id: "first-retained",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: "2026-02-15 09:00:00",)Fabricate(@@ -1720,7 +1727,6 @@ RSpec.describe Admin::DashboardController doip_address: "198.51.100.2",user_agent: firefox,session_id: "latest-retained",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: "2026-05-10 10:00:00",)@@ -1799,7 +1805,6 @@ RSpec.describe Admin::DashboardController doasn: 64_496,ip_address: "192.0.2.1",user_agent: chrome,- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: created_at,}end@@ -1887,7 +1892,7 @@ RSpec.describe Admin::DashboardController doendcontext "when the selected date range exceeds retention and traffic reaches the cap" do- let(:event_attributes) { { asn: 64_496, source: BrowserPageviewEvent::SOURCE_BEACON } }+ let(:event_attributes) { { asn: 64_496 } }let!(:first_retained) doFabricate(:browser_pageview_event,diff --git a/spec/requests/admin/reports_controller_spec.rb b/spec/requests/admin/reports_controller_spec.rbindex 3a19cfefe6f..0069014b547 100644--- a/spec/requests/admin/reports_controller_spec.rb+++ b/spec/requests/admin/reports_controller_spec.rb@@ -616,27 +616,24 @@ RSpec.describe Admin::ReportsController doendcontext "with browser pageview reports" do- it "lets an admin run them only when persist_browser_pageview_events is enabled" do- sign_in(admin)+ let(:browser_pageview_reports) do+ %w[top_countries_by_browser_pageviews top_entry_urls top_referrers_by_browser_pageviews]+ end- SiteSetting.persist_browser_pageview_events = false- Report::BROWSER_PAGEVIEW_REPORTS.each do |report_type|- get "/admin/reports/#{report_type}.json"- expect(response.status).to eq(404)- end+ it "lets an admin run them" do+ sign_in(admin)+ SiteSetting.use_legacy_pageviews = false- SiteSetting.persist_browser_pageview_events = true- Report::BROWSER_PAGEVIEW_REPORTS.each do |report_type|+ browser_pageview_reports.each do |report_type|get "/admin/reports/#{report_type}.json"expect(response.status).to eq(200)endend- it "denies a moderator even when persist_browser_pageview_events is enabled" do- SiteSetting.persist_browser_pageview_events = true+ it "denies a moderator" dosign_in(moderator)- Report::BROWSER_PAGEVIEW_REPORTS.each do |report_type|+ browser_pageview_reports.each do |report_type|get "/admin/reports/#{report_type}.json"expect(response.status).to eq(404)enddiff --git a/spec/requests/application_controller_spec.rb b/spec/requests/application_controller_spec.rbindex 1e8a70da93d..32eb44fb579 100644--- a/spec/requests/application_controller_spec.rb+++ b/spec/requests/application_controller_spec.rb@@ -1240,8 +1240,6 @@ RSpec.describe ApplicationController doMiddleware::AnonymousCache.enable_anon_cacheMiddleware::AnonymousCache.clear_all_cache!- SiteSetting.trigger_browser_pageview_events = true-get "/latest"expect(response.headers["X-Discourse-Cached"]).to eq("store")diff --git a/spec/services/admin_dashboard_site_traffic_explorer_spec.rb b/spec/services/admin_dashboard_site_traffic_explorer_spec.rbindex 3edc2b1a3e9..3dd27c64df1 100644--- a/spec/services/admin_dashboard_site_traffic_explorer_spec.rb+++ b/spec/services/admin_dashboard_site_traffic_explorer_spec.rb@@ -60,7 +60,6 @@ RSpec.describe AdminDashboardSiteTrafficExplorer doasn: 64_496,ip_address: "192.0.2.#{index + 1}",session_id: "browser-#{index}",- source: BrowserPageviewEvent::SOURCE_BEACON,browser:,language: languages[index],created_at: Time.zone.local(2026, 5, 10, 10, index),@@ -71,9 +70,7 @@ RSpec.describe AdminDashboardSiteTrafficExplorer dobefore dofreeze_time(Time.zone.local(2026, 5, 14, 12, 0, 0))SiteSetting.improved_crawler_detection = true- SiteSetting.persist_browser_pageview_events = trueSiteSetting.use_legacy_pageviews = false- BrowserPageviewEvent.stubs(:beacon_cutover_date).returns(Date.new(2026, 1, 1))endcontext "when the contract is invalid" do@@ -156,7 +153,6 @@ RSpec.describe AdminDashboardSiteTrafficExplorer donormalized_referrer: "other.example",normalized_referrer_version: BrowserPageviewEventUrlNormalizer::REFERRER_VERSION,session_id: "other",- source: BrowserPageviewEvent::SOURCE_BEACON,browser: :chrome,created_at: Time.zone.local(2026, 5, 10, 11),)@@ -239,7 +235,6 @@ RSpec.describe AdminDashboardSiteTrafficExplorer doasn: 64_500,ip_address: "198.51.100.1",session_id: "other-network",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: Time.zone.local(2026, 5, 10, 11),)DiscourseIpInfo@@ -295,7 +290,6 @@ RSpec.describe AdminDashboardSiteTrafficExplorer dourl: "/new-pageview",ip_address: "198.51.100.1",session_id: "new-pageview",- source: BrowserPageviewEvent::SOURCE_BEACON,user_agent: "ExampleBrowser/1.0",created_at: Time.zone.local(2026, 5, 11, 10),)@@ -310,7 +304,6 @@ RSpec.describe AdminDashboardSiteTrafficExplorer dourl: "/outside-range-entry",normalized_referrer: "external.example/path",session_id: "continuing-session",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: Time.zone.local(2026, 4, 30, 23, 59),)Fabricate(@@ -318,7 +311,6 @@ RSpec.describe AdminDashboardSiteTrafficExplorer dourl: "/inside-range-continuation",normalized_referrer: "test.localhost/internal",session_id: "continuing-session",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: Time.zone.local(2026, 5, 10, 12),)@@ -338,14 +330,12 @@ RSpec.describe AdminDashboardSiteTrafficExplorer do:browser_pageview_event,normalized_referrer: "external.example?article=traffic",session_id: "external-referrer-query",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: Time.zone.local(2026, 5, 10, 12),)Fabricate(:browser_pageview_event,normalized_referrer: "test.localhost?view=latest",session_id: "local-referrer-query",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: Time.zone.local(2026, 5, 10, 13),)expect(result.traffic.dig(:dimensions, "referrers")).to eq(@@ -382,7 +372,6 @@ RSpec.describe AdminDashboardSiteTrafficExplorer dourl: "/capped-session-entry",normalized_referrer: "external.example/path",session_id: "capped-session",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: Time.zone.local(2026, 5, 11, 10),)Fabricate(@@ -390,7 +379,6 @@ RSpec.describe AdminDashboardSiteTrafficExplorer dourl: "/capped-session-continuation",normalized_referrer: "test.localhost/internal",session_id: "capped-session",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: Time.zone.local(2026, 5, 11, 11),)@@ -413,7 +401,6 @@ RSpec.describe AdminDashboardSiteTrafficExplorer dourl: "/acquisition-at-timestamp",normalized_referrer: "external.example/path",session_id: "same-timestamp",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at:,)Fabricate(@@ -421,7 +408,6 @@ RSpec.describe AdminDashboardSiteTrafficExplorer dourl: "/internal-at-timestamp",normalized_referrer: "test.localhost/acquisition-at-timestamp",session_id: "same-timestamp",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at:,)diff --git a/spec/services/admin_dashboard_site_traffic_spec.rb b/spec/services/admin_dashboard_site_traffic_spec.rbindex af0a46f0140..25012786290 100644--- a/spec/services/admin_dashboard_site_traffic_spec.rb+++ b/spec/services/admin_dashboard_site_traffic_spec.rb@@ -6,7 +6,7 @@ RSpec.describe AdminDashboardSiteTraffic dobefore dofreeze_time(Time.zone.local(2026, 5, 14, 12, 0, 0))SiteSetting.use_legacy_pageviews = false- SiteSetting.persist_browser_pageview_events = false+ allow(Report).to receive(:hidden?).and_return(true)enddef build_traffic(start_date: nil, end_date: nil, guardian: admin.guardian)@@ -17,6 +17,10 @@ RSpec.describe AdminDashboardSiteTraffic do{ x: date, y: count }end+ def empty_session_kpis+ { bounce_rate: { value: nil }, average_session_duration_seconds: { value: nil } }+ end+def traffic_series(id, data, req: traffic_series_req(id))canonical_req = traffic_series_req(id)@@ -42,10 +46,6 @@ RSpec.describe AdminDashboardSiteTraffic doresponse[:pageview_series].find { |traffic_series| traffic_series[:req] == req }[:data]end- def use_beacon_cutover_date(date)- BrowserPageviewEvent.stubs(:beacon_cutover_date).returns(date)- end-describe ".build" doit "returns public-community KPIs and pageview series for selected dates" doSiteSetting.embed_topics_list = true@@ -62,6 +62,7 @@ RSpec.describe AdminDashboardSiteTraffic doexpect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")).to eq(kpis: {+ **empty_session_kpis,browser_pageviews: {value: 30,percent_change: 900,@@ -118,6 +119,7 @@ RSpec.describe AdminDashboardSiteTraffic doexpect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")).to eq(kpis: {+ **empty_session_kpis,browser_pageviews: {value: 5,percent_change: -75,@@ -187,12 +189,60 @@ RSpec.describe AdminDashboardSiteTraffic do)end+ it "switches to beacon counters after the first day with beacon data" do+ Fabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 20)+ Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-01", count: 15)+ Fabricate(:logged_in_browser_application_request, date: "2026-05-02", count: 30)+ Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-02", count: 25)+ Fabricate(:logged_in_browser_application_request, date: "2026-05-03", count: 40)++ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")++ expect(traffic_series_data(response, :logged_in)).to eq(+ [+ traffic_point("2026-05-01", 20),+ traffic_point("2026-05-02", 25),+ traffic_point("2026-05-03", 0),+ ],+ )+ end++ it "includes the first day of traffic on beacon-only sites" do+ Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-01", count: 15)+ Fabricate(:anonymous_browser_beacon_application_request, date: "2026-05-01", count: 25)++ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")++ expect(traffic_series_data(response, :logged_in)).to eq([traffic_point("2026-05-01", 15)])+ expect(traffic_series_data(response, :anonymous)).to eq([traffic_point("2026-05-01", 25)])+ end++ it "keeps using legacy counters when beacon data exists" do+ SiteSetting.use_legacy_pageviews = true++ Fabricate(:logged_in_browser_beacon_application_request, date: "2026-04-30", count: 100)+ Fabricate(:logged_in_application_request, date: "2026-05-01", count: 11)+ Fabricate(:anonymous_application_request, date: "2026-05-01", count: 22)+ Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-01", count: 110)+ Fabricate(:anonymous_browser_beacon_application_request, date: "2026-05-01", count: 220)++ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")++ expect(traffic_series_data(response, :logged_in, req: "page_view_logged_in")).to eq(+ [traffic_point("2026-05-01", 11)],+ )+ expect(traffic_series_data(response, :anonymous, req: "page_view_anon")).to eq(+ [traffic_point("2026-05-01", 22)],+ )+ end+it "omits trend data when the comparison period has no pageviews" doFabricate(:logged_in_browser_application_request, date: "2026-04-01", count: 1)Fabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 8)expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")[:kpis]).to eq(+ **empty_session_kpis,browser_pageviews: {value: 8,},@@ -208,6 +258,7 @@ RSpec.describe AdminDashboardSiteTraffic doFabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 8)expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")[:kpis]).to eq(+ **empty_session_kpis,browser_pageviews: {value: 8,},@@ -223,6 +274,7 @@ RSpec.describe AdminDashboardSiteTraffic doFabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 200_001)expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")[:kpis]).to eq(+ **empty_session_kpis,browser_pageviews: {value: 200_001,},@@ -238,6 +290,7 @@ RSpec.describe AdminDashboardSiteTraffic doFabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 10_050)expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")[:kpis]).to eq(+ **empty_session_kpis,browser_pageviews: {value: 10_050,percent_change: 0.5,@@ -258,6 +311,7 @@ RSpec.describe AdminDashboardSiteTraffic doFabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 110)expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")[:kpis]).to eq(+ **empty_session_kpis,browser_pageviews: {value: 110,percent_change: 10,@@ -272,120 +326,9 @@ RSpec.describe AdminDashboardSiteTraffic do)end- it "excludes mobile and beacon browser pageviews from totals and series" do- Fabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 10)- Fabricate(:anonymous_browser_application_request, date: "2026-05-01", count: 20)-- Fabricate(:logged_in_browser_mobile_application_request, date: "2026-05-01", count: 100)- Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-01", count: 200)- Fabricate(:anonymous_browser_mobile_application_request, date: "2026-05-01", count: 300)- Fabricate(:anonymous_browser_beacon_application_request, date: "2026-05-01", count: 400)-- expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")).to eq(- kpis: {- browser_pageviews: {- value: 30,- },- logged_in_share: {- value: 33,- },- },- pageview_series: [- traffic_series(:logged_in, [traffic_point("2026-05-01", 10)]),- traffic_series(:anonymous, [traffic_point("2026-05-01", 20)]),- traffic_series(:crawlers, [traffic_point("2026-05-01", 0)]),- ],- )- end-- it "uses beacon browser pageviews starting the day after dashboard improvements was enabled" do- use_beacon_cutover_date(Date.new(2026, 5, 3))-- Fabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 10)- Fabricate(:anonymous_browser_application_request, date: "2026-05-01", count: 20)- Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-01", count: 100)- Fabricate(:anonymous_browser_beacon_application_request, date: "2026-05-01", count: 200)-- Fabricate(:logged_in_browser_application_request, date: "2026-05-02", count: 11)- Fabricate(:anonymous_browser_application_request, date: "2026-05-02", count: 21)- Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-02", count: 110)- Fabricate(:anonymous_browser_beacon_application_request, date: "2026-05-02", count: 210)-- Fabricate(:logged_in_browser_application_request, date: "2026-05-03", count: 12)- Fabricate(:anonymous_browser_application_request, date: "2026-05-03", count: 22)- Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-03", count: 120)- Fabricate(:anonymous_browser_beacon_application_request, date: "2026-05-03", count: 220)-- expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")).to eq(- kpis: {- browser_pageviews: {- value: 402,- },- logged_in_share: {- value: 35,- },- },- pageview_series: [- traffic_series(- :logged_in,- [- traffic_point("2026-05-01", 10),- traffic_point("2026-05-02", 11),- traffic_point("2026-05-03", 120),- ],- ),- traffic_series(- :anonymous,- [- traffic_point("2026-05-01", 20),- traffic_point("2026-05-02", 21),- traffic_point("2026-05-03", 220),- ],- ),- traffic_series(- :crawlers,- [- traffic_point("2026-05-01", 0),- traffic_point("2026-05-02", 0),- traffic_point("2026-05-03", 0),- ],- ),- ],- )- end-- it "compares beacon pageviews against piggyback pageviews when the prior period predates the cutover" do- use_beacon_cutover_date(Date.new(2026, 5, 1))-- Fabricate(:logged_in_browser_beacon_application_request, date: "2026-04-28", count: 5)-- Fabricate(:logged_in_browser_application_request, date: "2026-04-29", count: 30)- Fabricate(:anonymous_browser_application_request, date: "2026-04-29", count: 70)- Fabricate(:logged_in_browser_beacon_application_request, date: "2026-04-29", count: 1)- Fabricate(:anonymous_browser_beacon_application_request, date: "2026-04-29", count: 2)-- Fabricate(:logged_in_browser_beacon_application_request, date: "2026-05-02", count: 60)- Fabricate(:anonymous_browser_beacon_application_request, date: "2026-05-02", count: 140)- Fabricate(:logged_in_browser_application_request, date: "2026-05-02", count: 3)- Fabricate(:anonymous_browser_application_request, date: "2026-05-02", count: 4)-- expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")[:kpis]).to eq(- browser_pageviews: {- value: 200,- percent_change: 100,- comparison_period: {- start_date: "2026-04-28",- end_date: "2026-04-30",- },- },- logged_in_share: {- value: 30,- },- )- end-- it "uses legacy human counters when legacy pageviews are enabled" do+ it "uses legacy human counters with improved crawler detection" doSiteSetting.use_legacy_pageviews = true+ SiteSetting.improved_crawler_detection = trueFabricate(:logged_in_application_request, date: "2026-05-01", count: 11)Fabricate(:anonymous_application_request, date: "2026-05-01", count: 22)@@ -397,6 +340,7 @@ RSpec.describe AdminDashboardSiteTraffic doexpect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")).to eq(kpis: {+ **empty_session_kpis,browser_pageviews: {value: 33,},@@ -407,6 +351,7 @@ RSpec.describe AdminDashboardSiteTraffic dopageview_series: [traffic_series(:logged_in, [traffic_point("2026-05-01", 11)], req: "page_view_logged_in"),traffic_series(:anonymous, [traffic_point("2026-05-01", 22)], req: "page_view_anon"),+ traffic_series(:likely_crawlers, [traffic_point("2026-05-01", 0)]),traffic_series(:crawlers, [traffic_point("2026-05-01", 4)]),],)@@ -417,6 +362,7 @@ RSpec.describe AdminDashboardSiteTraffic doexpect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")).to eq(kpis: {+ **empty_session_kpis,browser_pageviews: {value: 0,},@@ -436,6 +382,7 @@ RSpec.describe AdminDashboardSiteTraffic doexpect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")).to eq(kpis: {+ **empty_session_kpis,browser_pageviews: {value: 0,},@@ -460,6 +407,7 @@ RSpec.describe AdminDashboardSiteTraffic doexpect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")).to eq(kpis: {+ **empty_session_kpis,browser_pageviews: {value: 0,},@@ -488,6 +436,7 @@ RSpec.describe AdminDashboardSiteTraffic doexpect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")).to eq(kpis: {+ **empty_session_kpis,browser_pageviews: {value: 9,},@@ -503,6 +452,7 @@ RSpec.describe AdminDashboardSiteTraffic doexpect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")).to eq(kpis: {+ **empty_session_kpis,browser_pageviews: {value: 8,},@@ -544,6 +494,7 @@ RSpec.describe AdminDashboardSiteTraffic doexpect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")).to eq(kpis: {+ **empty_session_kpis,browser_pageviews: {value: 0,},@@ -624,7 +575,7 @@ RSpec.describe AdminDashboardSiteTraffic doendcontext "for top countries, top referrers, and top entry URLs" do- before { SiteSetting.persist_browser_pageview_events = true }+ before { allow(Report).to receive(:hidden?).and_call_original }def aggregate_rollupsrange = { start_date: 1.year.ago.to_date, end_date: Date.current }@@ -633,15 +584,6 @@ RSpec.describe AdminDashboardSiteTraffic doBrowserPageviewEntryUrlDailyRollup.aggregate(**range)end- it "omits browser pageview cards when persist_browser_pageview_events is disabled" do- SiteSetting.persist_browser_pageview_events = false-- result = build_traffic(start_date: nil, end_date: nil)- expect(result).not_to have_key(:top_countries)- expect(result).not_to have_key(:top_referrers)- expect(result).not_to have_key(:top_entry_urls)- end-it "omits top_countries and top_referrers when legacy pageviews are enabled" doSiteSetting.use_legacy_pageviews = true@@ -801,8 +743,6 @@ RSpec.describe AdminDashboardSiteTraffic doendcontext "for direct traffic share" do- before { SiteSetting.persist_browser_pageview_events = true }-def aggregate_referrer_rollupsBrowserPageviewReferrerDailyRollup.aggregate(start_date: "2026-05-01".to_date,@@ -946,24 +886,6 @@ RSpec.describe AdminDashboardSiteTraffic do)end- it "omits direct traffic when persist_browser_pageview_events is disabled" do- SiteSetting.persist_browser_pageview_events = false-- 3.times do- Fabricate(:browser_pageview_event, normalized_referrer: nil, created_at: "2026-05-01")- end- aggregate_referrer_rollups-- expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")[:kpis]).to eq(- browser_pageviews: {- value: 0,- },- logged_in_share: {- value: 0,- },- )- end-it "omits direct traffic when no pageviews were tracked in the period" doexpect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")[:kpis]).to eq(browser_pageviews: {@@ -983,8 +905,6 @@ RSpec.describe AdminDashboardSiteTraffic doendcontext "for bounce rate and average session duration" do- before { SiteSetting.persist_browser_pageview_events = true }-it "returns bounce rate and average session duration summed across the audience" doFabricate(:browser_pageview_session_engagement_daily_rollup,@@ -1167,27 +1087,6 @@ RSpec.describe AdminDashboardSiteTraffic do},)end-- it "omits the KPIs entirely when persist_browser_pageview_events is off" do- SiteSetting.persist_browser_pageview_events = false- Fabricate(- :browser_pageview_session_engagement_daily_rollup,- date: Date.new(2026, 5, 10),- logged_in: false,- sessions: 8,- bounced: 3,- engaged_seconds_total: 240,- )-- expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-14")[:kpis]).to eq(- browser_pageviews: {- value: 0,- },- logged_in_share: {- value: 0,- },- )- endendcontext "for likely crawlers" dodiff --git a/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb b/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rbindex dc8c24e65b7..79bd0bde2af 100644--- a/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb+++ b/spec/system/admin_dashboard_redesign/site_traffic_explorer_spec.rb@@ -8,9 +8,7 @@ RSpec.describe "Admin Dashboard Redesign | Site Traffic Explorer" dobefore doSiteSetting.dashboard_improvements = trueSiteSetting.improved_crawler_detection = true- SiteSetting.persist_browser_pageview_events = trueSiteSetting.use_legacy_pageviews = false- BrowserPageviewEvent.stubs(:beacon_cutover_date).returns(Date.new(2026, 1, 1))Discourse.stubs(:current_hostname).returns("test.localhost")DiscourseIpInfo.stubs(:get)@@ -55,7 +53,6 @@ RSpec.describe "Admin Dashboard Redesign | Site Traffic Explorer" dourl: "/crawler-shaped-traffic",session_id: "crawler-shaped-session",score: CrawlerScorer::BOT_SCORE_THRESHOLD + 1,- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: "2026-05-10 10:00:00",)@@ -72,7 +69,6 @@ RSpec.describe "Admin Dashboard Redesign | Site Traffic Explorer" do:browser_pageview_event,url: "/engaged-session",session_id: "engaged-session",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: "2026-05-14 10:00:00",)Fabricate(@@ -84,7 +80,6 @@ RSpec.describe "Admin Dashboard Redesign | Site Traffic Explorer" do:browser_pageview_event,url: "/active-session",session_id: "active-session",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: "2026-05-14 11:50:00",)Fabricate(@@ -107,20 +102,17 @@ RSpec.describe "Admin Dashboard Redesign | Site Traffic Explorer" do:browser_pageview_event,user_id: admin.id,session_id: "logged-in-session",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: "2026-05-10 10:00:00",)Fabricate(:browser_pageview_event,session_id: "anonymous-session",- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: "2026-05-11 10:00:00",)Fabricate(:browser_pageview_event,session_id: "crawler-session",score: CrawlerScorer::BOT_SCORE_THRESHOLD + 1,- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: "2026-05-12 10:00:00",)@@ -199,9 +191,7 @@ RSpec.describe "Admin Dashboard Redesign | Site Traffic Explorer" docreated_at: "2026-05-12 10:00:00",},]- events.each do |attributes|- Fabricate(:browser_pageview_event, source: BrowserPageviewEvent::SOURCE_BEACON, **attributes)- end+ events.each { |attributes| Fabricate(:browser_pageview_event, **attributes) }Fabricate(:browser_pageview_session_engagement,@@ -429,7 +419,6 @@ RSpec.describe "Admin Dashboard Redesign | Site Traffic Explorer" do:browser_pageview_event,url: path,topic_id: topic.id,- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: Time.zone.local(2026, 5, 10, 10, index, event_index),)end@@ -484,7 +473,6 @@ RSpec.describe "Admin Dashboard Redesign | Site Traffic Explorer" donormalized_referrer: normalized_referrer,normalized_referrer_version: BrowserPageviewEventUrlNormalizer::REFERRER_VERSION,session_id: session_id,- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: "2026-05-10 10:00:00",)end@@ -589,7 +577,6 @@ RSpec.describe "Admin Dashboard Redesign | Site Traffic Explorer" do:browser_pageview_event,url: url,session_id: url.delete_prefix("/"),- source: BrowserPageviewEvent::SOURCE_BEACON,created_at: created_at,)enddiff --git a/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb b/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rbindex ccc240aa689..da366a19879 100644--- a/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb+++ b/spec/system/admin_dashboard_redesign/site_traffic_section_spec.rb@@ -165,10 +165,7 @@ describe "Admin Dashboard Redesign | Site Traffic section" doendcontext "with top countries, top referrers, and top entry URLs cards" do- let(:browser_pageview_source) { BrowserPageviewEvent::SOURCE_BEACON }-before do- SiteSetting.persist_browser_pageview_events = trueUpcomingChangeEvent.create!(upcoming_change_name: "dashboard_improvements",event_type: :manual_opt_in,@@ -178,18 +175,6 @@ describe "Admin Dashboard Redesign | Site Traffic section" doDiscourse.cache.clearend- it "does not show the cards when persist_browser_pageview_events is off" do- SiteSetting.persist_browser_pageview_events = false-- dashboard.visit- traffic = dashboard.site_traffic-- expect(traffic).to have_no_top_countries_card- expect(traffic).to have_no_top_referrers_card- expect(traffic).to have_no_top_entry_urls_card- expect(traffic).to have_no_metric("Direct traffic")- end-it "shows ranked top countries and top referrers when events exist in the period",time: Time.zone.local(2026, 5, 14, 12, 0, 0) do2.times do@@ -198,7 +183,6 @@ describe "Admin Dashboard Redesign | Site Traffic section" docountry_code: "US",normalized_referrer: "news.ycombinator.com/item?id=42",created_at: "2026-05-12",- source: browser_pageview_source,)endFabricate(@@ -206,14 +190,12 @@ describe "Admin Dashboard Redesign | Site Traffic section" docountry_code: "GB",normalized_referrer: "reddit.com/r/discourse",created_at: "2026-05-12",- source: browser_pageview_source,)Fabricate(:browser_pageview_event,country_code: "DE",normalized_referrer: nil,created_at: "2026-05-12",- source: browser_pageview_source,)# Internal-referrer and direct (no-referrer) pageviews must not dilute the# top referrers percent denominator (it counts external referrer traffic only).@@ -223,7 +205,6 @@ describe "Admin Dashboard Redesign | Site Traffic section" docountry_code: "DE",normalized_referrer: "test.localhost/t/topic/1",created_at: "2026-05-12",- source: browser_pageview_source,)end@@ -312,7 +293,6 @@ describe "Admin Dashboard Redesign | Site Traffic section" do:browser_pageview_event,normalized_referrer: "news.ycombinator.com/item?id=42",created_at: "2026-05-12",- source: browser_pageview_source,)BrowserPageviewReferrerDailyRollup.aggregate(start_date: "2026-05-01".to_date,@@ -329,12 +309,7 @@ describe "Admin Dashboard Redesign | Site Traffic section" doit "drills into the full top countries report scoped to the dashboard period",time: Time.zone.local(2026, 5, 14, 12, 0, 0) do- Fabricate(- :browser_pageview_event,- country_code: "US",- created_at: "2026-05-12",- source: browser_pageview_source,- )+ Fabricate(:browser_pageview_event, country_code: "US", created_at: "2026-05-12")BrowserPageviewCountryDailyRollup.aggregate(start_date: "2026-05-01".to_date,end_date: "2026-05-14".to_date,@@ -350,8 +325,6 @@ describe "Admin Dashboard Redesign | Site Traffic section" doendcontext "with bounce rate and average session duration metrics" do- before { SiteSetting.persist_browser_pageview_events = true }-it "shows staff the bounce rate and average session duration for the period",time: Time.zone.local(2026, 5, 14, 12, 0, 0) doFabricate(@@ -391,24 +364,5 @@ describe "Admin Dashboard Redesign | Site Traffic section" do"Shown once visits are recorded for this period.",)end-- it "does not show the metric tiles when persist_browser_pageview_events is off",- time: Time.zone.local(2026, 5, 14, 12, 0, 0) do- SiteSetting.persist_browser_pageview_events = false- Fabricate(- :browser_pageview_session_engagement_daily_rollup,- date: Date.new(2026, 5, 12),- logged_in: false,- sessions: 8,- bounced: 3,- engaged_seconds_total: 480,- )-- dashboard.visit- traffic = dashboard.site_traffic-- expect(traffic).to have_no_bounce_rate- expect(traffic).to have_no_average_session_duration- endendenddiff --git a/spec/system/detect_human_activity_spec.rb b/spec/system/detect_human_activity_spec.rbindex 09551a2d1b9..3ebb67110c0 100644--- a/spec/system/detect_human_activity_spec.rb+++ b/spec/system/detect_human_activity_spec.rb@@ -4,10 +4,7 @@ describe "Detect human activity" dofab!(:topics) { Fabricate.times(5, :post).map(&:topic) }let(:discovery) { PageObjects::Pages::Discovery.new }- before do- SiteSetting.dashboard_improvements = true- SiteSetting.persist_browser_pageview_events = true- end+ before { SiteSetting.dashboard_improvements = true }def flush_engagementpage.driver.with_playwright_page do |pw_page|diff --git a/spec/system/request_tracker_spec.rb b/spec/system/request_tracker_spec.rbindex b578b206a01..49fe944a1fb 100644--- a/spec/system/request_tracker_spec.rb+++ b/spec/system/request_tracker_spec.rb@@ -6,8 +6,6 @@ describe "Request tracking" doApplicationRequest.enableCachedCounting.resetCachedCounting.enable- SiteSetting.trigger_browser_pageview_events = true- SiteSetting.persist_browser_pageview_events = falseendafter do@@ -18,10 +16,16 @@ describe "Request tracking" dolet(:pageview_tracking) { PageObjects::Pages::PageviewTracking.new }+ def track_pageview_events+ last_event_id = BrowserPageviewEvent.maximum(:id) || 0+ yield+ BrowserPageviewEvent.where("id > ?", last_event_id).order(:id).to_a+ end+describe "pageviews" doit "tracks an anonymous visit correctly" doevents =- DiscourseEvent.track_events(:browser_pageview) do+ track_pageview_events dovisit "/"try_until_success doCachedCounting.flush@@ -29,28 +33,30 @@ describe "Request tracking" doApplicationRequest.stats.slice("page_view_anon_total","page_view_anon_browser_total",+ "page_view_anon_browser_beacon_total","page_view_logged_in_total","page_view_crawler_total",),).to eq("page_view_anon_total" => 1,- "page_view_anon_browser_total" => 1,+ "page_view_anon_browser_total" => 0,+ "page_view_anon_browser_beacon_total" => 1,"page_view_logged_in_total" => 0,"page_view_crawler_total" => 0,)endend- event = events[0][:params].last+ event = events.first- expect(event[:user_id]).to be_nil- expect(event[:url]).to eq("#{Discourse.base_url_no_prefix}/")- expect(event[:ip_address]).to eq("::1")- expect(event[:referrer]).to be_blank- expect(event[:session_id]).to eq(pageview_tracking.session_id)+ expect(event.user_id).to be_nil+ expect(event.url).to eq("#{Discourse.base_url_no_prefix}/")+ expect(event.ip_address.to_s).to eq("::1")+ expect(event.referrer).to be_blank+ expect(event.session_id).to eq(pageview_tracking.session_id)events =- DiscourseEvent.track_events(:browser_pageview) do+ track_pageview_events dofind(".nav-item_categories a").clicktry_until_success do@@ -59,25 +65,27 @@ describe "Request tracking" doApplicationRequest.stats.slice("page_view_anon_total","page_view_anon_browser_total",+ "page_view_anon_browser_beacon_total","page_view_logged_in_total","page_view_crawler_total",),).to eq("page_view_anon_total" => 2,- "page_view_anon_browser_total" => 2,+ "page_view_anon_browser_total" => 1,+ "page_view_anon_browser_beacon_total" => 2,"page_view_logged_in_total" => 0,"page_view_crawler_total" => 0,)endend- event_2 = events[0][:params].last+ event_2 = events.first- expect(event_2[:user_id]).to be_nil- expect(event_2[:url]).to eq("#{Discourse.base_url_no_prefix}/categories")- expect(event_2[:ip_address]).to eq("::1")- expect(event_2[:referrer]).to eq("#{Discourse.base_url_no_prefix}/")- expect(event_2[:session_id]).to eq(event[:session_id])+ expect(event_2.user_id).to be_nil+ expect(event_2.url).to eq("#{Discourse.base_url_no_prefix}/categories")+ expect(event_2.ip_address.to_s).to eq("::1")+ expect(event_2.referrer).to eq("#{Discourse.base_url_no_prefix}/")+ expect(event_2.session_id).to eq(event.session_id)endit "tracks a crawler visit correctly" do@@ -85,7 +93,7 @@ describe "Request tracking" doSiteSetting.crawler_user_agents += "|chrome"events =- DiscourseEvent.track_events(:browser_pageview) do+ track_pageview_events dovisit "/"try_until_success do@@ -94,12 +102,14 @@ describe "Request tracking" doApplicationRequest.stats.slice("page_view_anon_total","page_view_anon_browser_total",+ "page_view_anon_browser_beacon_total","page_view_logged_in_total","page_view_crawler_total",),).to eq("page_view_anon_total" => 0,"page_view_anon_browser_total" => 0,+ "page_view_anon_browser_beacon_total" => 0,"page_view_logged_in_total" => 0,"page_view_crawler_total" => 1,)@@ -114,7 +124,7 @@ describe "Request tracking" dosign_in userevents =- DiscourseEvent.track_events(:browser_pageview) do+ track_pageview_events dovisit "/"try_until_success do@@ -126,28 +136,31 @@ describe "Request tracking" do"page_view_logged_in_total","page_view_crawler_total","page_view_logged_in_browser_total",+ "page_view_logged_in_browser_beacon_total",),).to eq("page_view_anon_total" => 0,"page_view_anon_browser_total" => 0,"page_view_logged_in_total" => 1,"page_view_crawler_total" => 0,- "page_view_logged_in_browser_total" => 1,+ "page_view_logged_in_browser_total" => 0,+ "page_view_logged_in_browser_beacon_total" => 1,)endend- event = events[0][:params].last+ event = events.first- expect(event[:user_id]).to eq(user.id)- expect(event[:url]).to eq("#{Discourse.base_url_no_prefix}/")- expect(event[:ip_address]).to eq("::1")- expect(event[:referrer]).to be_blank- expect(event[:session_id]).to be_present- expect(event[:topic_id]).to be_blank+ expect(event.user_id).to eq(user.id)+ expect(event.url).to eq("#{Discourse.base_url_no_prefix}/")+ expect(event.ip_address.to_s).to eq("::1")+ expect(event.referrer).to be_blank+ expect(event.session_id).to be_present+ expect(event.topic_id).to be_blank+ expect(event.language).to eq(page.evaluate_script("navigator.language"))events =- DiscourseEvent.track_events(:browser_pageview) do+ track_pageview_events dofind(".nav-item_categories a").clicktry_until_success doCachedCounting.flush@@ -158,31 +171,33 @@ describe "Request tracking" do"page_view_logged_in_total","page_view_crawler_total","page_view_logged_in_browser_total",+ "page_view_logged_in_browser_beacon_total",),).to eq("page_view_anon_total" => 0,"page_view_anon_browser_total" => 0,"page_view_logged_in_total" => 2,"page_view_crawler_total" => 0,- "page_view_logged_in_browser_total" => 2,+ "page_view_logged_in_browser_total" => 1,+ "page_view_logged_in_browser_beacon_total" => 2,)endend- event_2 = events[0][:params].last+ event_2 = events.first- expect(event_2[:user_id]).to eq(user.id)- expect(event_2[:url]).to eq("#{Discourse.base_url_no_prefix}/categories")- expect(event_2[:ip_address]).to eq("::1")- expect(event_2[:referrer]).to eq("#{Discourse.base_url_no_prefix}/")- expect(event_2[:session_id]).to eq(event[:session_id])+ expect(event_2.user_id).to eq(user.id)+ expect(event_2.url).to eq("#{Discourse.base_url_no_prefix}/categories")+ expect(event_2.ip_address.to_s).to eq("::1")+ expect(event_2.referrer).to eq("#{Discourse.base_url_no_prefix}/")+ expect(event_2.session_id).to eq(event.session_id)endit "tracks normal error pages correctly" doSiteSetting.bootstrap_error_pages = falseevents =- DiscourseEvent.track_events(:browser_pageview) do+ track_pageview_events dovisit "/foobar"try_until_success do@@ -194,6 +209,7 @@ describe "Request tracking" do"http_4xx_total","page_view_anon_total","page_view_anon_browser_total",+ "page_view_anon_browser_beacon_total","page_view_logged_in_total","page_view_crawler_total",),@@ -201,6 +217,7 @@ describe "Request tracking" do"http_4xx_total" => 1,"page_view_anon_total" => 0,"page_view_anon_browser_total" => 0,+ "page_view_anon_browser_beacon_total" => 0,"page_view_logged_in_total" => 0,"page_view_crawler_total" => 0,)@@ -218,13 +235,15 @@ describe "Request tracking" do"http_4xx_total","page_view_anon_total","page_view_anon_browser_total",+ "page_view_anon_browser_beacon_total","page_view_logged_in_total","page_view_crawler_total",),).to eq("http_4xx_total" => 1,"page_view_anon_total" => 1,- "page_view_anon_browser_total" => 1,+ "page_view_anon_browser_total" => 0,+ "page_view_anon_browser_beacon_total" => 1,"page_view_logged_in_total" => 0,"page_view_crawler_total" => 0,)@@ -233,7 +252,7 @@ describe "Request tracking" doit "tracks non-ember pages correctly" doevents =- DiscourseEvent.track_events(:browser_pageview) do+ track_pageview_events dovisit "/safe-mode"try_until_success do@@ -244,25 +263,27 @@ describe "Request tracking" doApplicationRequest.stats.slice("page_view_anon_total","page_view_anon_browser_total",+ "page_view_anon_browser_beacon_total","page_view_logged_in_total","page_view_crawler_total",),).to eq("page_view_anon_total" => 1,- "page_view_anon_browser_total" => 1,+ "page_view_anon_browser_total" => 0,+ "page_view_anon_browser_beacon_total" => 1,"page_view_logged_in_total" => 0,"page_view_crawler_total" => 0,)endend- event = events[0][:params].last+ event = events.first- expect(event[:user_id]).to be_nil- expect(event[:url]).to eq("#{Discourse.base_url_no_prefix}/safe-mode")- expect(event[:ip_address]).to eq("::1")- expect(event[:referrer]).to be_blank- expect(event[:session_id]).to be_present+ expect(event.user_id).to be_nil+ expect(event.url).to eq("#{Discourse.base_url_no_prefix}/safe-mode")+ expect(event.ip_address.to_s).to eq("::1")+ expect(event.referrer).to be_blank+ expect(event.session_id).to be_presentendit "tracks bootstrapped error pages correctly" do@@ -279,6 +300,7 @@ describe "Request tracking" do"http_4xx_total","page_view_anon_total","page_view_anon_browser_total",+ "page_view_anon_browser_beacon_total","page_view_logged_in_total","page_view_crawler_total",),@@ -286,6 +308,7 @@ describe "Request tracking" do"http_4xx_total" => 1,"page_view_anon_total" => 0,"page_view_anon_browser_total" => 0,+ "page_view_anon_browser_beacon_total" => 0,"page_view_logged_in_total" => 0,"page_view_crawler_total" => 0,)@@ -300,6 +323,7 @@ describe "Request tracking" do"http_4xx_total","page_view_anon_total","page_view_anon_browser_total",+ "page_view_anon_browser_beacon_total","page_view_logged_in_total","page_view_crawler_total",),@@ -307,6 +331,7 @@ describe "Request tracking" do"http_4xx_total" => 1,"page_view_anon_total" => 1,"page_view_anon_browser_total" => 1,+ "page_view_anon_browser_beacon_total" => 0,"page_view_logged_in_total" => 0,"page_view_crawler_total" => 0,)@@ -318,7 +343,7 @@ describe "Request tracking" doFabricate(:published_page, public: true, slug: "some-page", topic: Fabricate(:post).topic)events =- DiscourseEvent.track_events(:browser_pageview) do+ track_pageview_events dovisit "/pub/some-page"try_until_success do@@ -329,232 +354,27 @@ describe "Request tracking" doApplicationRequest.stats.slice("page_view_anon_total","page_view_anon_browser_total",+ "page_view_anon_browser_beacon_total","page_view_logged_in_total","page_view_crawler_total",),).to eq("page_view_anon_total" => 1,- "page_view_anon_browser_total" => 1,+ "page_view_anon_browser_total" => 0,+ "page_view_anon_browser_beacon_total" => 1,"page_view_logged_in_total" => 0,"page_view_crawler_total" => 0,)endend- event = events[0][:params].last+ event = events.first- expect(event[:user_id]).to be_nil- expect(event[:url]).to eq("#{Discourse.base_url_no_prefix}/pub/some-page")- expect(event[:ip_address]).to eq("::1")- expect(event[:referrer]).to be_blank- expect(event[:session_id]).to be_present- end-- context "when dashboard_improvements is enabled" do- before { SiteSetting.dashboard_improvements = true }-- it "tracks an anonymous visit correctly" do- all_events =- DiscourseEvent.track_events do |captured|- visit "/"- try_until_success do- CachedCounting.flush- expect(- ApplicationRequest.stats.slice(- "page_view_anon_total",- "page_view_anon_browser_total",- "page_view_anon_browser_beacon_total",- "page_view_logged_in_total",- "page_view_crawler_total",- ),- ).to eq(- "page_view_anon_total" => 1,- "page_view_anon_browser_total" => 1,- "page_view_anon_browser_beacon_total" => 1,- "page_view_logged_in_total" => 0,- "page_view_crawler_total" => 0,- )- expect(captured.count { |e| e[:event_name] == :beacon_browser_pageview }).to eq(1)- expect(captured.count { |e| e[:event_name] == :browser_pageview }).to eq(1)- end- end-- beacon_event =- all_events.find { |e| e[:event_name] == :beacon_browser_pageview }[:params].last- non_beacon_event = all_events.find { |e| e[:event_name] == :browser_pageview }[:params].last-- [beacon_event, non_beacon_event].each do |event|- expect(event[:user_id]).to be_nil- expect(event[:url]).to eq("#{Discourse.base_url_no_prefix}/")- expect(event[:ip_address]).to eq("::1")- expect(event[:referrer]).to be_blank- expect(event[:session_id]).to be_present- end-- expect(beacon_event[:session_id]).to eq(non_beacon_event[:session_id])-- all_events =- DiscourseEvent.track_events do- find(".nav-item_categories a").click-- CachedCounting.flush- expect(- ApplicationRequest.stats.slice(- "page_view_anon_total",- "page_view_anon_browser_total",- "page_view_anon_browser_beacon_total",- "page_view_logged_in_total",- "page_view_crawler_total",- ),- ).to eq(- "page_view_anon_total" => 2,- "page_view_anon_browser_total" => 2,- "page_view_anon_browser_beacon_total" => 2,- "page_view_logged_in_total" => 0,- "page_view_crawler_total" => 0,- )- end-- beacon_events_2 = all_events.select { |e| e[:event_name] == :beacon_browser_pageview }- non_beacon_events_2 = all_events.select { |e| e[:event_name] == :browser_pageview }-- expect(beacon_events_2.size).to eq(1)- expect(non_beacon_events_2.size).to eq(1)-- beacon_event_2 = beacon_events_2[0][:params].last- non_beacon_event_2 = non_beacon_events_2[0][:params].last-- [beacon_event_2, non_beacon_event_2].each do |event|- expect(event[:user_id]).to be_nil- expect(event[:url]).to eq("#{Discourse.base_url_no_prefix}/categories")- expect(event[:ip_address]).to eq("::1")- expect(event[:referrer]).to eq("#{Discourse.base_url}/")- expect(event[:session_id]).to eq(beacon_event[:session_id])- end- end-- it "tracks a logged-in session correctly" do- user = Fabricate(:user)- sign_in user-- all_events =- DiscourseEvent.track_events do |captured|- visit "/"-- try_until_success do- CachedCounting.flush- expect(- ApplicationRequest.stats.slice(- "page_view_anon_total",- "page_view_anon_browser_total",- "page_view_logged_in_total",- "page_view_logged_in_browser_total",- "page_view_logged_in_browser_beacon_total",- "page_view_crawler_total",- ),- ).to eq(- "page_view_anon_total" => 0,- "page_view_anon_browser_total" => 0,- "page_view_logged_in_total" => 1,- "page_view_logged_in_browser_total" => 1,- "page_view_logged_in_browser_beacon_total" => 1,- "page_view_crawler_total" => 0,- )- expect(captured.count { |e| e[:event_name] == :beacon_browser_pageview }).to eq(1)- expect(captured.count { |e| e[:event_name] == :browser_pageview }).to eq(1)- end- end-- beacon_event =- all_events.find { |e| e[:event_name] == :beacon_browser_pageview }[:params].last- non_beacon_event = all_events.find { |e| e[:event_name] == :browser_pageview }[:params].last-- [beacon_event, non_beacon_event].each do |event|- expect(event[:user_id]).to eq(user.id)- expect(event[:url]).to eq("#{Discourse.base_url_no_prefix}/")- expect(event[:ip_address]).to eq("::1")- expect(event[:referrer]).to be_blank- expect(event[:session_id]).to be_present- expect(event[:topic_id]).to be_blank- end-- expect(beacon_event[:session_id]).to eq(non_beacon_event[:session_id])- expect(beacon_event[:language]).to eq(page.evaluate_script("navigator.language"))-- all_events =- DiscourseEvent.track_events do- find(".nav-item_categories a").click- CachedCounting.flush- expect(- ApplicationRequest.stats.slice(- "page_view_anon_total",- "page_view_anon_browser_total",- "page_view_logged_in_total",- "page_view_logged_in_browser_total",- "page_view_logged_in_browser_beacon_total",- "page_view_crawler_total",- ),- ).to eq(- "page_view_anon_total" => 0,- "page_view_anon_browser_total" => 0,- "page_view_logged_in_total" => 2,- "page_view_logged_in_browser_total" => 2,- "page_view_logged_in_browser_beacon_total" => 2,- "page_view_crawler_total" => 0,- )- end-- beacon_events_2 = all_events.select { |e| e[:event_name] == :beacon_browser_pageview }- non_beacon_events_2 = all_events.select { |e| e[:event_name] == :browser_pageview }-- expect(beacon_events_2.size).to eq(1)- expect(non_beacon_events_2.size).to eq(1)-- beacon_event_2 = beacon_events_2[0][:params].last- non_beacon_event_2 = non_beacon_events_2[0][:params].last-- [beacon_event_2, non_beacon_event_2].each do |event|- expect(event[:user_id]).to eq(user.id)- expect(event[:url]).to eq("#{Discourse.base_url_no_prefix}/categories")- expect(event[:ip_address]).to eq("::1")- expect(event[:referrer]).to eq("#{Discourse.base_url}/")- expect(event[:session_id]).to eq(beacon_event[:session_id])- end- end-- it "tracks a crawler visit correctly" do- SiteSetting.crawler_user_agents += "|chrome"-- all_events =- DiscourseEvent.track_events do- visit "/"-- try_until_success do- CachedCounting.flush- expect(- ApplicationRequest.stats.slice(- "page_view_anon_total",- "page_view_anon_browser_total",- "page_view_anon_browser_beacon_total",- "page_view_logged_in_total",- "page_view_crawler_total",- ),- ).to eq(- "page_view_anon_total" => 0,- "page_view_anon_browser_total" => 0,- "page_view_anon_browser_beacon_total" => 0,- "page_view_logged_in_total" => 0,- "page_view_crawler_total" => 1,- )- end- end-- beacon_events = all_events.select { |e| e[:event_name] == :beacon_browser_pageview }- non_beacon_events = all_events.select { |e| e[:event_name] == :browser_pageview }-- expect(beacon_events).to be_blank- expect(non_beacon_events).to be_blank- end+ expect(event.user_id).to be_nil+ expect(event.url).to eq("#{Discourse.base_url_no_prefix}/pub/some-page")+ expect(event.ip_address.to_s).to eq("::1")+ expect(event.referrer).to be_blank+ expect(event.session_id).to be_presentendend@@ -566,45 +386,11 @@ describe "Request tracking" docontext "when logged in" dobefore { sign_in(current_user) }- it "tracks user viewing a topic correctly with deferred tracking" do- events =- DiscourseEvent.track_events(:browser_pageview) do |captured_events|- visit topic.url-- try_until_success do- CachedCounting.flush- expect(TopicViewItem.exists?(topic_id: topic.id, user_id: current_user.id)).to eq(- true,- )- expect(- TopicViewStat.exists?(- topic_id: topic.id,- viewed_at: Time.zone.now.to_date,- anonymous_views: 0,- logged_in_views: 1,- ),- ).to eq(true)- expect(- captured_events.any? { |event| event[:event_name] == :browser_pageview },- ).to eq(true)- end- end-- event = events[0][:params].last-- expect(event[:user_id]).to eq(current_user.id)- expect(event[:url]).to eq(topic.url)- expect(event[:ip_address]).to eq("::1")- expect(event[:referrer]).to be_blank- expect(event[:session_id]).to be_present- expect(event[:topic_id]).to eq(topic.id)- end-- it "tracks user viewing a topic correctly with explicit tracking" do+ it "tracks a logged-in topic view during in-app navigation" dovisit "/"events =- DiscourseEvent.track_events(:browser_pageview) do+ track_pageview_events dofind(".topic-list-item .raw-topic-link[data-topic-id='#{topic.id}']").clicktry_until_success do@@ -623,56 +409,24 @@ describe "Request tracking" doendend- # Find the event for the topic we navigated to (by topic_id) to avoid flakiness- # from potential timing issues with multiple events- event = events.find { |e| e[:params].last[:topic_id] == topic.id }&.dig(:params)&.last+ event = events.find { |pageview_event| pageview_event.topic_id == topic.id }expect(event).to be_present- expect(event[:user_id]).to eq(current_user.id)- expect(event[:url]).to eq(topic.url)- expect(event[:ip_address]).to eq("::1")- expect(event[:referrer]).to eq("#{Discourse.base_url_no_prefix}/")- expect(event[:session_id]).to be_present+ expect(event.user_id).to eq(current_user.id)+ expect(event.url).to eq(topic.url)+ expect(event.ip_address.to_s).to eq("::1")+ expect(event.referrer).to eq("#{Discourse.base_url_no_prefix}/")+ expect(event.session_id).to be_present+ expect(event.language).to eq(page.evaluate_script("navigator.language"))endendcontext "when anonymous" do- it "tracks an anonymous user viewing a topic correctly with deferred tracking" do- events =- DiscourseEvent.track_events(:browser_pageview) do |captured_events|- visit topic.url-- try_until_success do- CachedCounting.flush- expect(TopicViewItem.exists?(topic_id: topic.id, user_id: nil)).to eq(true)- expect(- TopicViewStat.exists?(- topic_id: topic.id,- viewed_at: Time.zone.now.to_date,- anonymous_views: 1,- logged_in_views: 0,- ),- ).to eq(true)- expect(- captured_events.any? { |event| event[:event_name] == :browser_pageview },- ).to eq(true)- end- end-- event = events[0][:params].last-- expect(event[:user_id]).to be_blank- expect(event[:url]).to eq(topic.url)- expect(event[:ip_address]).to eq("::1")- expect(event[:referrer]).to be_blank- expect(event[:session_id]).to be_present- end-- it "tracks an anonymous user viewing a topic correctly with explicit tracking" do+ it "tracks an anonymous topic view during in-app navigation" dovisit "/"events =- DiscourseEvent.track_events(:browser_pageview) do+ track_pageview_events dofind(".topic-list-item .raw-topic-link[data-topic-id='#{topic.id}']").clicktry_until_success do@@ -689,122 +443,21 @@ describe "Request tracking" doendend- # Find the event for the topic we navigated to (by topic_id) to avoid flakiness- # from potential timing issues with multiple events- event = events.find { |e| e[:params].last[:topic_id] == topic.id }&.dig(:params)&.last+ event = events.find { |pageview_event| pageview_event.topic_id == topic.id }expect(event).to be_present- expect(event[:user_id]).to be_blank- expect(event[:url]).to eq(topic.url)- expect(event[:ip_address]).to eq("::1")- expect(event[:referrer]).to eq("#{Discourse.base_url}/")- expect(event[:session_id]).to be_present+ expect(event.user_id).to be_blank+ expect(event.url).to eq(topic.url)+ expect(event.ip_address.to_s).to eq("::1")+ expect(event.referrer).to eq("#{Discourse.base_url}/")+ expect(event.session_id).to be_presentendend- context "when dashboard_improvements is enabled" do- before { SiteSetting.dashboard_improvements = true }-- context "when logged in" do- before { sign_in(current_user) }-- it "tracks topic views correctly via beacon" do- visit "/"-- all_events =- DiscourseEvent.track_events do- find(".topic-list-item .raw-topic-link[data-topic-id='#{topic.id}']").click-- CachedCounting.flush- expect(TopicViewItem.exists?(topic_id: topic.id, user_id: current_user.id)).to eq(- true,- )- expect(- TopicViewStat.exists?(- topic_id: topic.id,- viewed_at: Time.zone.now.to_date,- anonymous_views: 0,- logged_in_views: 1,- ),- ).to eq(true)- end-- beacon_event =- all_events- .select { |e| e[:event_name] == :beacon_browser_pageview }- .find { |e| e[:params].last[:topic_id] == topic.id }- &.dig(:params)- &.last- non_beacon_event =- all_events- .select { |e| e[:event_name] == :browser_pageview }- .find { |e| e[:params].last[:topic_id] == topic.id }- &.dig(:params)- &.last-- expect(beacon_event).to be_present- expect(non_beacon_event).to be_present-- [beacon_event, non_beacon_event].each do |event|- expect(event[:user_id]).to eq(current_user.id)- expect(event[:url]).to eq(topic.url)- expect(event[:ip_address]).to eq("::1")- expect(event[:referrer]).to eq("#{Discourse.base_url}/")- expect(event[:session_id]).to be_present- expect(event[:topic_id]).to eq(topic.id)- end-- expect(beacon_event[:language]).to eq(page.evaluate_script("navigator.language"))- end- end-+ context "when navigating with beacons" docontext "when anonymous" dolet(:discovery) { PageObjects::Pages::Discovery.new }- it "tracks topic views correctly via beacon" do- visit "/"-- all_events =- DiscourseEvent.track_events do- find(".topic-list-item .raw-topic-link[data-topic-id='#{topic.id}']").click-- CachedCounting.flush- expect(TopicViewItem.exists?(topic_id: topic.id, user_id: nil)).to eq(true)- expect(- TopicViewStat.exists?(- topic_id: topic.id,- viewed_at: Time.zone.now.to_date,- anonymous_views: 1,- logged_in_views: 0,- ),- ).to eq(true)- end-- beacon_event =- all_events- .select { |event| event[:event_name] == :beacon_browser_pageview }- .find { |event| event[:params].last[:topic_id] == topic.id }- &.dig(:params)- &.last- non_beacon_event =- all_events- .select { |event| event[:event_name] == :browser_pageview }- .find { |event| event[:params].last[:topic_id] == topic.id }- &.dig(:params)- &.last-- expect(beacon_event).to be_present- expect(non_beacon_event).to be_present-- [beacon_event, non_beacon_event].each do |event|- expect(event[:user_id]).to be_blank- expect(event[:url]).to eq(topic.url)- expect(event[:ip_address]).to eq("::1")- expect(event[:referrer]).to eq("#{Discourse.base_url}/")- expect(event[:session_id]).to be_present- end- end-it "tracks the previous URL as referrer on browser back and forward navigation via beacon" dovisit "/"wait_for_beacon_count(1)@@ -813,38 +466,36 @@ describe "Request tracking" dowait_for_beacon_count(2)events =- DiscourseEvent.track_events(:beacon_browser_pageview) do+ track_pageview_events dopage.go_backwait_for_beacon_count(3)end- beacon_back_event = events.first[:params].last+ beacon_back_event = events.first- expect(beacon_back_event[:url]).to eq("#{Discourse.base_url_no_prefix}/")- expect(beacon_back_event[:referrer]).to eq(topic.url)+ expect(beacon_back_event.url).to eq("#{Discourse.base_url_no_prefix}/")+ expect(beacon_back_event.referrer).to eq(topic.url)events =- DiscourseEvent.track_events(:beacon_browser_pageview) do+ track_pageview_events dopage.go_forwardwait_for_beacon_count(4)end- beacon_forward_event = events.first[:params].last+ beacon_forward_event = events.first- expect(beacon_forward_event[:url]).to eq(topic.url)- expect(beacon_forward_event[:referrer]).to eq("#{Discourse.base_url}/")+ expect(beacon_forward_event.url).to eq(topic.url)+ expect(beacon_forward_event.referrer).to eq("#{Discourse.base_url}/")end- it "tracks pageviews via both trackers when sorting a topic list" do+ it "tracks browser pageviews when sorting a topic list" dovisit "/latest"wait_for_beacon_count(1)- wait_for_browser_count(1)discovery.topic_list_header.click_sort_by("views")expect(page).to have_current_path("/latest?ascending=false&order=views")wait_for_beacon_count(2)- wait_for_browser_count(2)enddef wait_for_beacon_count(count)@@ -853,13 +504,6 @@ describe "Request tracking" doexpect(ApplicationRequest.stats["page_view_anon_browser_beacon_total"]).to eq(count)endend-- def wait_for_browser_count(count)- try_until_success do- CachedCounting.flush- expect(ApplicationRequest.stats["page_view_anon_browser_total"]).to eq(count)- end- endendendend@@ -869,10 +513,7 @@ describe "Request tracking" dofab!(:topic)fab!(:post) { Fabricate(:post, topic: topic) }- before do- SiteSetting.dashboard_improvements = true- Middleware::RequestTracker.bpv_notifications_enabled = true- end+ before { Middleware::RequestTracker.bpv_notifications_enabled = true }after { Middleware::RequestTracker.bpv_notifications_enabled = false }@@ -892,26 +533,14 @@ describe "Request tracking" do}end- shared_examples "logs piggyback and beacon entries on home and topic" do- it "writes piggyback and beacon entries on each navigation" do+ shared_examples "logs beacon entries on home and topic" do+ it "writes a beacon entry on each navigation" dohome_url = "#{Discourse.base_url_no_prefix}/"home_entries =- capture_log_entries(controller: "PageviewController", entries: 2) { visit "/" }-- home_piggyback = home_entries.find { |e| e["action"] == "piggyback" }- home_beacon = home_entries.find { |e| e["action"] == "beacon" }+ capture_log_entries(controller: "PageviewController", entries: 1) { visit "/" }- expect(home_piggyback).to include(- common_fields(- controller: "PageviewController",- action: "piggyback",- path: "/pageview",- username: expected_username,- url: home_url,- ),- )- expect(home_beacon).to include(+ expect(home_entries.first).to include(common_fields(controller: "PageviewController",action: "beacon",@@ -922,25 +551,11 @@ describe "Request tracking" do)topic_entries =- capture_log_entries(controller: "PageviewController", entries: 2) do+ capture_log_entries(controller: "PageviewController", entries: 1) dofind(".topic-list-item .raw-topic-link[data-topic-id='#{topic.id}']").clickend- topic_piggyback = topic_entries.find { |e| e["action"] == "piggyback" }- topic_beacon = topic_entries.find { |e| e["action"] == "beacon" }-- expect(topic_piggyback).to include(- common_fields(- controller: "PageviewController",- action: "piggyback",- path: "/pageview",- username: expected_username,- url: topic.url,- ),- "topic_id" => topic.id,- "referrer" => home_url,- )- expect(topic_beacon).to include(+ expect(topic_entries.first).to include(common_fields(controller: "PageviewController",action: "beacon",@@ -952,15 +567,16 @@ describe "Request tracking" do"referrer" => home_url,)- all_session_ids = (home_entries + topic_entries).map { |e| e["session_id"] }.uniq- expect(all_session_ids.size).to eq(1)+ expect((home_entries + topic_entries).map { |entry| entry["session_id"] }.uniq.size).to eq(+ 1,+ )endendcontext "when anonymous" dolet(:expected_username) { nil }- include_examples "logs piggyback and beacon entries on home and topic"+ include_examples "logs beacon entries on home and topic"endcontext "when logged in" do@@ -968,7 +584,7 @@ describe "Request tracking" dolet(:expected_username) { user.username }- include_examples "logs piggyback and beacon entries on home and topic"+ include_examples "logs beacon entries on home and topic"endendenddiff --git a/spec/system/standalone_scripts_spec.rb b/spec/system/standalone_scripts_spec.rbindex 22b117fe4db..190444d58c8 100644--- a/spec/system/standalone_scripts_spec.rb+++ b/spec/system/standalone_scripts_spec.rb@@ -98,12 +98,12 @@ describe "Standalone scripts" doenddescribe "pageview.js" do- it "sends a pageview tracking request on a non-ember page" do+ it "sends a beacon pageview tracking request on a non-ember page" dopageview_requests = []page.driver.with_playwright_page do |pw_page|pw_page.on("request",- ->(request) { pageview_requests << request.url if request.url.include?("/pageview") },+ ->(request) { pageview_requests << request.url if request.url.end_with?("/srv/pv") },)end@@ -119,9 +119,7 @@ describe "Standalone scripts" dopage.driver.with_playwright_page do |pw_page|pw_page.on("request",- lambda do |request|- tracking_requests << request.url if request.url.match?(%r{/(pageview|srv/pv)$})- end,+ lambda { |request| tracking_requests << request.url if request.url.end_with?("/srv/pv") },)end