A platform for community discussion. Free, open, simple.
DEV: Log browser page view tracking requests to rails logs (#39600)
Alan Guo Xiang Tan4922c72ea706
diff --git a/app/controllers/pageview_controller.rb b/app/controllers/pageview_controller.rbdeleted file mode 100644index 013e9e45af3..00000000000--- a/app/controllers/pageview_controller.rb+++ /dev/null@@ -1,14 +0,0 @@-# frozen_string_literal: true--class PageviewController < ApplicationController- skip_before_action :check_xhr,- :redirect_to_login_if_required,- :redirect_to_profile_if_required,- :preload_json,- :verify_authenticity_token-- def index- # pageview tracking is handled by middleware- render plain: "ok"- end-enddiff --git a/config/initializers/101-lograge.rb b/config/initializers/101-lograge.rbindex 4dff76aed98..32e9770da36 100644--- a/config/initializers/101-lograge.rb+++ b/config/initializers/101-lograge.rb@@ -1,6 +1,8 @@# frozen_string_literal: true-if ENV["ENABLE_LOGSTASH_LOGGER"] == "1"+require "discourse_lograge"++if DiscourseLograge.enabled?require "lograge"Rails.application.config.after_initialize do@@ -79,7 +81,7 @@ if ENV["ENABLE_LOGSTASH_LOGGER"] == "1"nilend- { ip: ip, username: username }+ DiscourseLograge.custom_payload(ip: ip, username: username)rescue => eRails.logger.warn("Failed to append custom payload: #{e.message}\n#{e.backtrace.join("\n")}",@@ -172,8 +174,10 @@ if ENV["ENABLE_LOGSTASH_LOGGER"] == "1"Rails.logger.broadcasts.find { |logger| logger.is_a?(ActiveSupport::Logger) },)- Logster.logger.subscribe do |severity, message, progname, opts, &block|- config.lograge.logger.add_with_opts(severity, message, progname, opts, &block)+ if Logster.logger+ Logster.logger.subscribe do |severity, message, progname, opts, &block|+ config.lograge.logger.add_with_opts(severity, message, progname, opts, &block)+ endendendenddiff --git a/config/routes.rb b/config/routes.rbindex 53b0b358859..909082c20a0 100644--- a/config/routes.rb+++ b/config/routes.rb@@ -1928,8 +1928,6 @@ Discourse::Application.routes.draw doresources :sidebar_sections, only: %i[index create update destroy]put "/sidebar_sections/reset/:id" => "sidebar_sections#reset"- post "/pageview" => "pageview#index"-get "/form-templates/:id" => "form_templates#show"get "/form-templates" => "form_templates#index"diff --git a/lib/discourse_lograge.rb b/lib/discourse_lograge.rbnew file mode 100644index 00000000000..df9ab8d58f2--- /dev/null+++ b/lib/discourse_lograge.rb@@ -0,0 +1,11 @@+# frozen_string_literal: true++module DiscourseLograge+ def self.enabled?+ ENV["ENABLE_LOGSTASH_LOGGER"] == "1"+ end++ def self.custom_payload(ip:, username:, **extras)+ { ip: ip, username: username, **extras.compact }+ end+enddiff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rbindex 1c888646759..3ca470926ae 100644--- a/lib/middleware/request_tracker.rb+++ b/lib/middleware/request_tracker.rb@@ -3,10 +3,20 @@require "method_profiler"require "middleware/anonymous_cache"require "http_user_agent_encoder"+require "discourse_lograge"class Middleware::RequestTracker@@detailed_request_loggers = nil@@ip_skipper = nil+ @@bpv_notifications_enabled = !Rails.env.test?++ def self.bpv_notifications_enabled+ @@bpv_notifications_enabled+ end++ def self.bpv_notifications_enabled=(value)+ @@bpv_notifications_enabled = value+ end# You can add exceptions to our app rate limiter in the app.yml ENV section.# example:@@ -105,57 +115,35 @@ class Middleware::RequestTrackerendend- if data[:browser_page_view] && !data[:is_crawler] && !data[:is_beacon]- if data[:has_auth_cookie]- ApplicationRequest.increment!(:page_view_logged_in_browser)- ApplicationRequest.increment!(:page_view_logged_in_browser_mobile) if data[:is_mobile]-- trigger_browser_pageview_event(data)-- if data[:topic_id].present? && data[:current_user_id].present?- TopicsController.defer_topic_view(- data[:topic_id],- data[:request_remote_ip],- data[:current_user_id],- )- end- elsif !SiteSetting.login_required- ApplicationRequest.increment!(:page_view_anon_browser)- ApplicationRequest.increment!(:page_view_anon_browser_mobile) if data[:is_mobile]-- trigger_browser_pageview_event(data)-- if data[:topic_id].present?- TopicsController.defer_topic_view(data[:topic_id], data[:request_remote_ip])- end- end- end-- if data[:is_beacon] && !data[:is_crawler]- if data[:has_auth_cookie]- ApplicationRequest.increment!(:page_view_logged_in_browser_beacon)- if data[:is_mobile]- ApplicationRequest.increment!(:page_view_logged_in_browser_mobile_beacon)+ if tracks_browser_page_view?(data)+ if data[:is_beacon]+ if data[:has_auth_cookie]+ ApplicationRequest.increment!(:page_view_logged_in_browser_beacon)+ if data[:is_mobile]+ ApplicationRequest.increment!(:page_view_logged_in_browser_mobile_beacon)+ end+ else+ ApplicationRequest.increment!(:page_view_anon_browser_beacon)+ ApplicationRequest.increment!(:page_view_anon_browser_mobile_beacon) if data[:is_mobile]end-trigger_beacon_browser_pageview_event(data)-- if data[:topic_id].present? && data[:current_user_id].present?- TopicsController.defer_topic_view(- data[:topic_id],- data[:request_remote_ip],- data[:current_user_id],- )+ else+ if data[:has_auth_cookie]+ ApplicationRequest.increment!(:page_view_logged_in_browser)+ ApplicationRequest.increment!(:page_view_logged_in_browser_mobile) if data[:is_mobile]+ else+ ApplicationRequest.increment!(:page_view_anon_browser)+ ApplicationRequest.increment!(:page_view_anon_browser_mobile) if data[:is_mobile]end- elsif !SiteSetting.login_required && !CrawlerDetection.crawler_ip?(data[:request_remote_ip])- ApplicationRequest.increment!(:page_view_anon_browser_beacon)- ApplicationRequest.increment!(:page_view_anon_browser_mobile_beacon) if data[:is_mobile]-- trigger_beacon_browser_pageview_event(data)+ trigger_browser_pageview_event(data)+ end- if data[:topic_id].present?- TopicsController.defer_topic_view(data[:topic_id], data[:request_remote_ip])- end+ if data[:topic_id].present? && (!data[:has_auth_cookie] || data[:current_user_id].present?)+ TopicsController.defer_topic_view(+ data[:topic_id],+ data[:request_remote_ip],+ data[:current_user_id],+ )endend@@ -205,25 +193,36 @@ class Middleware::RequestTrackeris_message_bus = request.path.start_with?("#{Discourse.base_path}/message-bus/")is_topic_timings = request.path.start_with?("#{Discourse.base_path}/topics/timings")- current_user_id =- if view_tracking_data[:deferred_track_view] || view_tracking_data[:explicit_track_view]- begin- (auth_cookie&.[](:user_id) || CurrentUser.lookup_from_env(env)&.id)- rescue Discourse::InvalidAccess => err- # This error is raised when the API key is invalid, no need to stop the show.- Discourse.warn_exception(- err,- message: "RequestTracker.get_data failed with an invalid API key error",- )- nil+ current_user_id = nil+ current_username = nil++ if view_tracking_data[:browser_page_view]+ begin+ if auth_cookie.is_a?(Hash)+ current_user_id = auth_cookie[:user_id]+ current_username = auth_cookie[:username]+ else+ user = CurrentUser.lookup_from_env(env)+ if user+ current_user_id = user.id+ current_username = user.username+ endend+ rescue Discourse::InvalidAccess => err+ # This error is raised when the API key is invalid, no need to stop the show.+ Discourse.warn_exception(+ err,+ message: "RequestTracker.get_data failed with an invalid API key error",+ )end+ endrequest_data = {status: status,is_crawler: helper.is_crawler?,has_auth_cookie: has_auth_cookie,current_user_id: current_user_id,+ current_username: current_username,is_api: is_api,is_user_api: is_user_api,is_background: is_message_bus || is_topic_timings,@@ -285,7 +284,7 @@ class Middleware::RequestTracker@@detailed_request_loggers.each { |logger| logger.call(env, data) }end- log_later(data)+ log_later(data, env, request)endend@@ -342,7 +341,8 @@ class Middleware::RequestTrackerenv["discourse.request_tracker"] = self- if self.class.is_beacon_tracking_request?(request)+ if self.class.is_beacon_tracking_request?(request) ||+ self.class.is_pageview_tracking_request?(request)result = [204, {}, []]return resultend@@ -402,10 +402,13 @@ class Middleware::RequestTrackerendend- def log_later(data)+ def log_later(data, env, request)Scheduler::Defer.later("Track view") dobegin- self.class.log_request(data) unless Discourse.pg_readonly_mode?+ unless Discourse.pg_readonly_mode?+ self.class.log_request(data)+ instrument_browser_page_view(env, request, data)+ endrescue ActiveRecord::ReadOnlyError# Just noop if ActiveRecord is preventing writesend@@ -614,11 +617,24 @@ class Middleware::RequestTracker}end+ def self.tracks_browser_page_view?(data)+ return false unless data[:browser_page_view]+ return false if data[:is_crawler]+ return true if data[:has_auth_cookie]+ return false if SiteSetting.login_required+ return false if data[:is_beacon] && CrawlerDetection.crawler_ip?(data[:request_remote_ip])+ true+ end+def self.is_beacon_tracking_request?(request)SiteSetting.use_beacon_for_browser_page_views && request.post? &&request.path == Discourse.beacon_pv_tracking_pathend+ def self.is_pageview_tracking_request?(request)+ request.post? && request.path == "#{Discourse.base_path}/pageview"+ end+def self.extract_beacon_view_tracking_data(env)body = env["rack.input"]&.readenv["rack.input"]&.rewind@@ -679,4 +695,46 @@ class Middleware::RequestTracker}endprivate_class_method :build_browser_pageview_event_payload++ private++ def instrument_browser_page_view(env, request, data)+ return unless data[:browser_page_view]+ return unless DiscourseLograge.enabled?+ return unless self.class.bpv_notifications_enabled++ request ||= Rack::Request.new(env)++ if data[:is_beacon]+ action = "beacon"+ path = request.fullpath+ else+ action = "piggyback"+ path = "#{Discourse.base_path}/pageview"+ end++ payload = {+ controller: "PageviewController",+ action: action,+ method: "POST",+ path: path,+ format: :json,+ status: 204,+ params: {+ },+ headers: ActionDispatch::Request.new(env).headers,+ custom_payload:+ DiscourseLograge.custom_payload(+ ip: data[:request_remote_ip],+ username: data[:current_username],+ tracked: self.class.tracks_browser_page_view?(data),+ url: data[:tracking_url],+ referrer: data[:tracking_referrer],+ session_id: data[:tracking_session_id],+ topic_id: data[:topic_id],+ ),+ }++ ActiveSupport::Notifications.instrument("process_action.action_controller", payload)+ endenddiff --git a/spec/rails_helper.rb b/spec/rails_helper.rbindex 4b7c832e76c..a8add1118e2 100644--- a/spec/rails_helper.rb+++ b/spec/rails_helper.rb@@ -101,6 +101,7 @@ class PlaywrightLoggerendENV["RAILS_ENV"] ||= "test"+ENV["ENABLE_LOGSTASH_LOGGER"] ||= "1"require File.expand_path("../../config/environment", __FILE__)Discourse.singleton_class.prepend(RspecWarnExceptionCapture)require "rspec/rails"diff --git a/spec/support/system_helpers.rb b/spec/support/system_helpers.rbindex b3c7a30cdbc..f9dac7df393 100644--- a/spec/support/system_helpers.rb+++ b/spec/support/system_helpers.rb@@ -414,6 +414,33 @@ module SystemHelpersdef html_translation_to_text(html_translation)Nokogiri.HTML5(html_translation).at("body").inner_textend++ def capture_log_entries(controller:, entries:, action: nil)+ log = Rails.root.join("log", "#{Rails.env}.log")+ File.truncate(log, 0) if File.exist?(log)++ yield++ read =+ lambda do+ return [] unless File.exist?(log)+ File.open(log) do |f|+ f+ .read+ .lines+ .reject { |l| l.strip.empty? }+ .filter_map do |line|+ JSON.parse(line)+ rescue JSON::ParserError+ nil+ end+ .select { |e| e["controller"] == controller && (action.nil? || e["action"] == action) }+ end+ end++ try_until_success { raise Capybara::ExpectationNotMet if read.call.size < entries }+ read.call+ endendmodule CapybaraSessionEmberWaiterdiff --git a/spec/system/request_tracker_spec.rb b/spec/system/request_tracker_spec.rbindex 8ebe888b2d6..5f556f3f5c1 100644--- a/spec/system/request_tracker_spec.rb+++ b/spec/system/request_tracker_spec.rb@@ -799,4 +799,111 @@ describe "Request tracking" doendendend++ describe "BPV log entries" do+ fab!(:user)+ fab!(:topic)+ fab!(:post) { Fabricate(:post, topic: topic) }++ before do+ SiteSetting.use_beacon_for_browser_page_views = true+ Middleware::RequestTracker.bpv_notifications_enabled = true+ end++ after { Middleware::RequestTracker.bpv_notifications_enabled = false }++ def common_fields(controller:, action:, path:, username:, url:)+ {+ "controller" => controller,+ "action" => action,+ "method" => "POST",+ "path" => path,+ "status" => 204,+ "format" => "json",+ "tracked" => true,+ "ip" => be_present,+ "url" => url,+ "session_id" => be_present,+ "username" => username ? eq(username) : be_blank,+ }+ end++ shared_examples "logs piggyback and beacon entries on home and topic" do+ it "writes piggyback and beacon entries on each navigation" do+ home_url = "#{Discourse.base_url_no_prefix}/"++ home_entries =+ capture_log_entries(controller: "PageviewController", entries: 2) { visit "/" }++ home_piggyback = home_entries.find { |e| e["action"] == "piggyback" }+ home_beacon = home_entries.find { |e| e["action"] == "beacon" }++ expect(home_piggyback).to include(+ common_fields(+ controller: "PageviewController",+ action: "piggyback",+ path: "/pageview",+ username: expected_username,+ url: home_url,+ ),+ )+ expect(home_beacon).to include(+ common_fields(+ controller: "PageviewController",+ action: "beacon",+ path: "/srv/pv",+ username: expected_username,+ url: home_url,+ ),+ )++ topic_entries =+ capture_log_entries(controller: "PageviewController", entries: 2) do+ find(".topic-list-item .raw-topic-link[data-topic-id='#{topic.id}']").click+ end++ topic_piggyback = topic_entries.find { |e| e["action"] == "piggyback" }+ topic_beacon = topic_entries.find { |e| e["action"] == "beacon" }++ expect(topic_piggyback).to include(+ common_fields(+ controller: "PageviewController",+ action: "piggyback",+ path: "/pageview",+ username: expected_username,+ url: topic.url,+ ),+ "topic_id" => topic.id,+ "referrer" => home_url,+ )+ expect(topic_beacon).to include(+ common_fields(+ controller: "PageviewController",+ action: "beacon",+ path: "/srv/pv",+ username: expected_username,+ url: topic.url,+ ),+ "topic_id" => topic.id,+ "referrer" => home_url,+ )++ all_session_ids = (home_entries + topic_entries).map { |e| e["session_id"] }.uniq+ expect(all_session_ids.size).to eq(1)+ end+ end++ context "when anonymous" do+ let(:expected_username) { nil }++ include_examples "logs piggyback and beacon entries on home and topic"+ end++ context "when logged in" do+ before { sign_in(user) }+ let(:expected_username) { user.username }++ include_examples "logs piggyback and beacon entries on home and topic"+ end+ endend