Skip to content

A platform for community discussion. Free, open, simple.

DEV: Log browser page view tracking requests to rails logs (#39600)

Alan Guo Xiang Tan4922c72ea706
diff --git a/app/controllers/pageview_controller.rb b/app/controllers/pageview_controller.rb
deleted file mode 100644
index 013e9e45af3..00000000000
--- a/app/controllers/pageview_controller.rb
+++ /dev/null
@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-class PageviewController < ApplicationController
- skip_before_action :check_xhr,
- :redirect_to_login_if_required,
- :redirect_to_profile_if_required,
- :preload_json,
- :verify_authenticity_token
-
- def index
- # pageview tracking is handled by middleware
- render plain: "ok"
- end
-end
diff --git a/config/initializers/101-lograge.rb b/config/initializers/101-lograge.rb
index 4dff76aed98..32e9770da36 100644
--- a/config/initializers/101-lograge.rb
+++ b/config/initializers/101-lograge.rb
@@ -1,6 +1,8 @@
# frozen_string_literal: true
-if ENV["ENABLE_LOGSTASH_LOGGER"] == "1"
+require "discourse_lograge"
+
+if DiscourseLograge.enabled?
require "lograge"
Rails.application.config.after_initialize do
@@ -79,7 +81,7 @@ if ENV["ENABLE_LOGSTASH_LOGGER"] == "1"
nil
end
- { ip: ip, username: username }
+ DiscourseLograge.custom_payload(ip: ip, username: username)
rescue => e
Rails.logger.warn(
"Failed to append custom payload: #{e.message}\n#{e.backtrace.join("\n")}",
@@ -172,8 +174,10 @@ if ENV["ENABLE_LOGSTASH_LOGGER"] == "1"
Rails.logger.broadcasts.find { |logger| logger.is_a?(ActiveSupport::Logger) },
)
- Logster.logger.subscribe do |severity, message, progname, opts, &block|
- config.lograge.logger.add_with_opts(severity, message, progname, opts, &block)
+ if Logster.logger
+ Logster.logger.subscribe do |severity, message, progname, opts, &block|
+ config.lograge.logger.add_with_opts(severity, message, progname, opts, &block)
+ end
end
end
end
diff --git a/config/routes.rb b/config/routes.rb
index 53b0b358859..909082c20a0 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -1928,8 +1928,6 @@ Discourse::Application.routes.draw do
resources :sidebar_sections, only: %i[index create update destroy]
put "/sidebar_sections/reset/:id" => "sidebar_sections#reset"
- post "/pageview" => "pageview#index"
-
get "/form-templates/:id" => "form_templates#show"
get "/form-templates" => "form_templates#index"
diff --git a/lib/discourse_lograge.rb b/lib/discourse_lograge.rb
new file mode 100644
index 00000000000..df9ab8d58f2
--- /dev/null
+++ b/lib/discourse_lograge.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module DiscourseLograge
+ def self.enabled?
+ ENV["ENABLE_LOGSTASH_LOGGER"] == "1"
+ end
+
+ def self.custom_payload(ip:, username:, **extras)
+ { ip: ip, username: username, **extras.compact }
+ end
+end
diff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rb
index 1c888646759..3ca470926ae 100644
--- a/lib/middleware/request_tracker.rb
+++ b/lib/middleware/request_tracker.rb
@@ -3,10 +3,20 @@
require "method_profiler"
require "middleware/anonymous_cache"
require "http_user_agent_encoder"
+require "discourse_lograge"
class Middleware::RequestTracker
@@detailed_request_loggers = nil
@@ip_skipper = nil
+ @@bpv_notifications_enabled = !Rails.env.test?
+
+ def self.bpv_notifications_enabled
+ @@bpv_notifications_enabled
+ end
+
+ def self.bpv_notifications_enabled=(value)
+ @@bpv_notifications_enabled = value
+ end
# You can add exceptions to our app rate limiter in the app.yml ENV section.
# example:
@@ -105,57 +115,35 @@ class Middleware::RequestTracker
end
end
- if data[:browser_page_view] && !data[:is_crawler] && !data[:is_beacon]
- if data[:has_auth_cookie]
- ApplicationRequest.increment!(:page_view_logged_in_browser)
- ApplicationRequest.increment!(:page_view_logged_in_browser_mobile) if data[:is_mobile]
-
- trigger_browser_pageview_event(data)
-
- if data[:topic_id].present? && data[:current_user_id].present?
- TopicsController.defer_topic_view(
- data[:topic_id],
- data[:request_remote_ip],
- data[:current_user_id],
- )
- end
- elsif !SiteSetting.login_required
- ApplicationRequest.increment!(:page_view_anon_browser)
- ApplicationRequest.increment!(:page_view_anon_browser_mobile) if data[:is_mobile]
-
- trigger_browser_pageview_event(data)
-
- if data[:topic_id].present?
- TopicsController.defer_topic_view(data[:topic_id], data[:request_remote_ip])
- end
- end
- end
-
- if data[:is_beacon] && !data[:is_crawler]
- if data[:has_auth_cookie]
- ApplicationRequest.increment!(:page_view_logged_in_browser_beacon)
- if data[:is_mobile]
- ApplicationRequest.increment!(:page_view_logged_in_browser_mobile_beacon)
+ if tracks_browser_page_view?(data)
+ if data[:is_beacon]
+ if data[:has_auth_cookie]
+ ApplicationRequest.increment!(:page_view_logged_in_browser_beacon)
+ if data[:is_mobile]
+ ApplicationRequest.increment!(:page_view_logged_in_browser_mobile_beacon)
+ end
+ else
+ ApplicationRequest.increment!(:page_view_anon_browser_beacon)
+ ApplicationRequest.increment!(:page_view_anon_browser_mobile_beacon) if data[:is_mobile]
end
-
trigger_beacon_browser_pageview_event(data)
-
- if data[:topic_id].present? && data[:current_user_id].present?
- TopicsController.defer_topic_view(
- data[:topic_id],
- data[:request_remote_ip],
- data[:current_user_id],
- )
+ else
+ if data[:has_auth_cookie]
+ ApplicationRequest.increment!(:page_view_logged_in_browser)
+ ApplicationRequest.increment!(:page_view_logged_in_browser_mobile) if data[:is_mobile]
+ else
+ ApplicationRequest.increment!(:page_view_anon_browser)
+ ApplicationRequest.increment!(:page_view_anon_browser_mobile) if data[:is_mobile]
end
- elsif !SiteSetting.login_required && !CrawlerDetection.crawler_ip?(data[:request_remote_ip])
- ApplicationRequest.increment!(:page_view_anon_browser_beacon)
- ApplicationRequest.increment!(:page_view_anon_browser_mobile_beacon) if data[:is_mobile]
-
- trigger_beacon_browser_pageview_event(data)
+ trigger_browser_pageview_event(data)
+ end
- if data[:topic_id].present?
- TopicsController.defer_topic_view(data[:topic_id], data[:request_remote_ip])
- end
+ if data[:topic_id].present? && (!data[:has_auth_cookie] || data[:current_user_id].present?)
+ TopicsController.defer_topic_view(
+ data[:topic_id],
+ data[:request_remote_ip],
+ data[:current_user_id],
+ )
end
end
@@ -205,25 +193,36 @@ class Middleware::RequestTracker
is_message_bus = request.path.start_with?("#{Discourse.base_path}/message-bus/")
is_topic_timings = request.path.start_with?("#{Discourse.base_path}/topics/timings")
- current_user_id =
- if view_tracking_data[:deferred_track_view] || view_tracking_data[:explicit_track_view]
- begin
- (auth_cookie&.[](:user_id) || CurrentUser.lookup_from_env(env)&.id)
- rescue Discourse::InvalidAccess => err
- # This error is raised when the API key is invalid, no need to stop the show.
- Discourse.warn_exception(
- err,
- message: "RequestTracker.get_data failed with an invalid API key error",
- )
- nil
+ current_user_id = nil
+ current_username = nil
+
+ if view_tracking_data[:browser_page_view]
+ begin
+ if auth_cookie.is_a?(Hash)
+ current_user_id = auth_cookie[:user_id]
+ current_username = auth_cookie[:username]
+ else
+ user = CurrentUser.lookup_from_env(env)
+ if user
+ current_user_id = user.id
+ current_username = user.username
+ end
end
+ rescue Discourse::InvalidAccess => err
+ # This error is raised when the API key is invalid, no need to stop the show.
+ Discourse.warn_exception(
+ err,
+ message: "RequestTracker.get_data failed with an invalid API key error",
+ )
end
+ end
request_data = {
status: status,
is_crawler: helper.is_crawler?,
has_auth_cookie: has_auth_cookie,
current_user_id: current_user_id,
+ current_username: current_username,
is_api: is_api,
is_user_api: is_user_api,
is_background: is_message_bus || is_topic_timings,
@@ -285,7 +284,7 @@ class Middleware::RequestTracker
@@detailed_request_loggers.each { |logger| logger.call(env, data) }
end
- log_later(data)
+ log_later(data, env, request)
end
end
@@ -342,7 +341,8 @@ class Middleware::RequestTracker
env["discourse.request_tracker"] = self
- if self.class.is_beacon_tracking_request?(request)
+ if self.class.is_beacon_tracking_request?(request) ||
+ self.class.is_pageview_tracking_request?(request)
result = [204, {}, []]
return result
end
@@ -402,10 +402,13 @@ class Middleware::RequestTracker
end
end
- def log_later(data)
+ def log_later(data, env, request)
Scheduler::Defer.later("Track view") do
begin
- self.class.log_request(data) unless Discourse.pg_readonly_mode?
+ unless Discourse.pg_readonly_mode?
+ self.class.log_request(data)
+ instrument_browser_page_view(env, request, data)
+ end
rescue ActiveRecord::ReadOnlyError
# Just noop if ActiveRecord is preventing writes
end
@@ -614,11 +617,24 @@ class Middleware::RequestTracker
}
end
+ def self.tracks_browser_page_view?(data)
+ return false unless data[:browser_page_view]
+ return false if data[:is_crawler]
+ return true if data[:has_auth_cookie]
+ return false if SiteSetting.login_required
+ return false if data[:is_beacon] && CrawlerDetection.crawler_ip?(data[:request_remote_ip])
+ true
+ end
+
def self.is_beacon_tracking_request?(request)
SiteSetting.use_beacon_for_browser_page_views && request.post? &&
request.path == Discourse.beacon_pv_tracking_path
end
+ def self.is_pageview_tracking_request?(request)
+ request.post? && request.path == "#{Discourse.base_path}/pageview"
+ end
+
def self.extract_beacon_view_tracking_data(env)
body = env["rack.input"]&.read
env["rack.input"]&.rewind
@@ -679,4 +695,46 @@ class Middleware::RequestTracker
}
end
private_class_method :build_browser_pageview_event_payload
+
+ private
+
+ def instrument_browser_page_view(env, request, data)
+ return unless data[:browser_page_view]
+ return unless DiscourseLograge.enabled?
+ return unless self.class.bpv_notifications_enabled
+
+ request ||= Rack::Request.new(env)
+
+ if data[:is_beacon]
+ action = "beacon"
+ path = request.fullpath
+ else
+ action = "piggyback"
+ path = "#{Discourse.base_path}/pageview"
+ end
+
+ payload = {
+ controller: "PageviewController",
+ action: action,
+ method: "POST",
+ path: path,
+ format: :json,
+ status: 204,
+ params: {
+ },
+ headers: ActionDispatch::Request.new(env).headers,
+ custom_payload:
+ DiscourseLograge.custom_payload(
+ ip: data[:request_remote_ip],
+ username: data[:current_username],
+ tracked: self.class.tracks_browser_page_view?(data),
+ url: data[:tracking_url],
+ referrer: data[:tracking_referrer],
+ session_id: data[:tracking_session_id],
+ topic_id: data[:topic_id],
+ ),
+ }
+
+ ActiveSupport::Notifications.instrument("process_action.action_controller", payload)
+ end
end
diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb
index 4b7c832e76c..a8add1118e2 100644
--- a/spec/rails_helper.rb
+++ b/spec/rails_helper.rb
@@ -101,6 +101,7 @@ class PlaywrightLogger
end
ENV["RAILS_ENV"] ||= "test"
+ENV["ENABLE_LOGSTASH_LOGGER"] ||= "1"
require File.expand_path("../../config/environment", __FILE__)
Discourse.singleton_class.prepend(RspecWarnExceptionCapture)
require "rspec/rails"
diff --git a/spec/support/system_helpers.rb b/spec/support/system_helpers.rb
index b3c7a30cdbc..f9dac7df393 100644
--- a/spec/support/system_helpers.rb
+++ b/spec/support/system_helpers.rb
@@ -414,6 +414,33 @@ module SystemHelpers
def html_translation_to_text(html_translation)
Nokogiri.HTML5(html_translation).at("body").inner_text
end
+
+ def capture_log_entries(controller:, entries:, action: nil)
+ log = Rails.root.join("log", "#{Rails.env}.log")
+ File.truncate(log, 0) if File.exist?(log)
+
+ yield
+
+ read =
+ lambda do
+ return [] unless File.exist?(log)
+ File.open(log) do |f|
+ f
+ .read
+ .lines
+ .reject { |l| l.strip.empty? }
+ .filter_map do |line|
+ JSON.parse(line)
+ rescue JSON::ParserError
+ nil
+ end
+ .select { |e| e["controller"] == controller && (action.nil? || e["action"] == action) }
+ end
+ end
+
+ try_until_success { raise Capybara::ExpectationNotMet if read.call.size < entries }
+ read.call
+ end
end
module CapybaraSessionEmberWaiter
diff --git a/spec/system/request_tracker_spec.rb b/spec/system/request_tracker_spec.rb
index 8ebe888b2d6..5f556f3f5c1 100644
--- a/spec/system/request_tracker_spec.rb
+++ b/spec/system/request_tracker_spec.rb
@@ -799,4 +799,111 @@ describe "Request tracking" do
end
end
end
+
+ describe "BPV log entries" do
+ fab!(:user)
+ fab!(:topic)
+ fab!(:post) { Fabricate(:post, topic: topic) }
+
+ before do
+ SiteSetting.use_beacon_for_browser_page_views = true
+ Middleware::RequestTracker.bpv_notifications_enabled = true
+ end
+
+ after { Middleware::RequestTracker.bpv_notifications_enabled = false }
+
+ def common_fields(controller:, action:, path:, username:, url:)
+ {
+ "controller" => controller,
+ "action" => action,
+ "method" => "POST",
+ "path" => path,
+ "status" => 204,
+ "format" => "json",
+ "tracked" => true,
+ "ip" => be_present,
+ "url" => url,
+ "session_id" => be_present,
+ "username" => username ? eq(username) : be_blank,
+ }
+ end
+
+ shared_examples "logs piggyback and beacon entries on home and topic" do
+ it "writes piggyback and beacon entries on each navigation" do
+ home_url = "#{Discourse.base_url_no_prefix}/"
+
+ home_entries =
+ capture_log_entries(controller: "PageviewController", entries: 2) { visit "/" }
+
+ home_piggyback = home_entries.find { |e| e["action"] == "piggyback" }
+ home_beacon = home_entries.find { |e| e["action"] == "beacon" }
+
+ expect(home_piggyback).to include(
+ common_fields(
+ controller: "PageviewController",
+ action: "piggyback",
+ path: "/pageview",
+ username: expected_username,
+ url: home_url,
+ ),
+ )
+ expect(home_beacon).to include(
+ common_fields(
+ controller: "PageviewController",
+ action: "beacon",
+ path: "/srv/pv",
+ username: expected_username,
+ url: home_url,
+ ),
+ )
+
+ topic_entries =
+ capture_log_entries(controller: "PageviewController", entries: 2) do
+ find(".topic-list-item .raw-topic-link[data-topic-id='#{topic.id}']").click
+ end
+
+ topic_piggyback = topic_entries.find { |e| e["action"] == "piggyback" }
+ topic_beacon = topic_entries.find { |e| e["action"] == "beacon" }
+
+ expect(topic_piggyback).to include(
+ common_fields(
+ controller: "PageviewController",
+ action: "piggyback",
+ path: "/pageview",
+ username: expected_username,
+ url: topic.url,
+ ),
+ "topic_id" => topic.id,
+ "referrer" => home_url,
+ )
+ expect(topic_beacon).to include(
+ common_fields(
+ controller: "PageviewController",
+ action: "beacon",
+ path: "/srv/pv",
+ username: expected_username,
+ url: topic.url,
+ ),
+ "topic_id" => topic.id,
+ "referrer" => home_url,
+ )
+
+ all_session_ids = (home_entries + topic_entries).map { |e| e["session_id"] }.uniq
+ expect(all_session_ids.size).to eq(1)
+ end
+ end
+
+ context "when anonymous" do
+ let(:expected_username) { nil }
+
+ include_examples "logs piggyback and beacon entries on home and topic"
+ end
+
+ context "when logged in" do
+ before { sign_in(user) }
+ let(:expected_username) { user.username }
+
+ include_examples "logs piggyback and beacon entries on home and topic"
+ end
+ end
end