Skip to content

A platform for community discussion. Free, open, simple.

10.3 KB
Raw Blame
# frozen_string_literal: true

class BrowserPageviewEvent < ActiveRecord::Base
  self.ignored_columns += %i[source] # TODO(02-2027): Remove after the column is dropped

  MAX_SESSION_ID_LENGTH = 32
  MAX_URL_LENGTH = 2000
  MAX_REFERRER_LENGTH = 2000
  MAX_USER_AGENT_LENGTH = 1000
  MAX_LANGUAGE_LENGTH = 255
  MAX_NORMALIZED_LANGUAGE_LENGTH = 13
  MAX_NORMALIZED_REFERRER_LENGTH = 2000
  MAX_NORMALIZED_URL_LENGTH = 2000
  RETENTION_PERIOD = 3.months
  BROWSER_UNKNOWN = 0
  BROWSERS = {
    unknown: BROWSER_UNKNOWN,
    chrome: 1,
    edge: 2,
    safari: 3,
    firefox: 4,
    opera: 5,
    ie: 6,
    samsung_browser: 7,
    uc_browser: 8,
    android_browser: 9,
    qq_browser: 10,
    baidu_browser: 11,
    kaios_browser: 12,
  }.freeze
  REDIS_QUEUE_KEY = "browser_pageview_events:pending"
  REDIS_FLUSH_LOCK_KEY = "browser_pageview_events:flush"
  REDIS_FLUSH_BATCH_SIZE = 1000
  REDIS_QUEUE_MAX_SIZE = 1_000_000
  REDIS_QUEUE_TTL = 1.day

  enum :browser, BROWSERS, prefix: true, scopes: false

  class << self
    def enqueue_for_later(payload)
      return unless valid_payload?(payload)
      return if Discourse.redis.llen(REDIS_QUEUE_KEY) >= REDIS_QUEUE_MAX_SIZE

      Discourse.redis.multi do |transaction|
        transaction.rpush(REDIS_QUEUE_KEY, JSON.generate(serialize_payload(payload)))
        transaction.expire(REDIS_QUEUE_KEY, REDIS_QUEUE_TTL)
      end
    rescue Redis::BaseConnectionError => e
      Rails.logger.warn("Failed to queue BrowserPageviewEvent in Redis: #{e.message}")
    end

    def create_from_payload!(payload)
      BrowserPageviewEvent.create!(attributes_from_payload(payload))
    end

    def flush_queued!
      return 0 if Discourse.pg_readonly_mode?

      processed = 0

      DistributedMutex.synchronize(REDIS_FLUSH_LOCK_KEY, validity: 5.minutes) do
        entries = Array(Discourse.redis.lrange(REDIS_QUEUE_KEY, 0, REDIS_FLUSH_BATCH_SIZE - 1))
        queued_attributes = []

        entries.each do |entry|
          payload = deserialize_payload(entry)
          # Old processes can leave piggyback events in Redis during deployment.
          queued_attributes << (payload[:source] == 1 ? nil : attributes_from_payload(payload))
          processed += 1
        rescue => e
          Rails.logger.error("Discarding queued BrowserPageviewEvent: #{e.message}")
          queued_attributes << nil
          processed += 1
        end

        begin
          insert_rows!(queued_attributes)
        rescue ActiveRecord::ReadOnlyError
          Discourse.received_postgres_readonly!
          return 0
        rescue ActiveRecord::StatementInvalid => e
          if postgres_readonly_error?(e)
            Discourse.received_postgres_readonly!
            return 0
          end

          return 0 if postgres_connection_error?(e)

          Rails.logger.error("Failed to insert queued BrowserPageviewEvents: #{e.message}")
          return 0
        end
      end

      processed
    end

    def queued_count
      Discourse.redis.llen(REDIS_QUEUE_KEY).to_i
    end

    def clear_queued!
      Discourse.redis.del(REDIS_QUEUE_KEY)
    end

    def postgres_readonly_error?(error)
      error.cause.is_a?(PG::ReadOnlySqlTransaction)
    end

    private

    def insert_rows!(queued_attributes)
      return if queued_attributes.blank?

      rows = queued_attributes.compact
      if rows.present?
        BrowserPageviewEvent.transaction(requires_new: true) do
          BrowserPageviewEvent.insert_all(rows, returning: false)
        end
      end
      Discourse.redis.ltrim(REDIS_QUEUE_KEY, queued_attributes.length, -1)
    rescue ActiveRecord::StatementInvalid => e
      Rails.logger.error(
        "Failed to insert queued BrowserPageviewEvents in bulk; retrying individually: #{e.message}",
      )
      insert_rows_individually!(queued_attributes)
    end

    def insert_rows_individually!(queued_attributes)
      queued_attributes.each do |attributes|
        if attributes
          BrowserPageviewEvent.transaction(requires_new: true) do
            BrowserPageviewEvent.insert_all([attributes], returning: false)
          end
        end
        Discourse.redis.ltrim(REDIS_QUEUE_KEY, 1, -1)
      rescue ActiveRecord::StatementInvalid => e
        raise if postgres_readonly_error?(e) || postgres_connection_error?(e)

        Rails.logger.error(
          "Discarding queued BrowserPageviewEvent after insert failure: #{e.message}",
        )
        Discourse.redis.ltrim(REDIS_QUEUE_KEY, 1, -1)
      end
    end

    def serialize_payload(payload)
      payload = payload.with_indifferent_access

      {
        user_id: payload[:user_id],
        url: payload[:url]&.slice(0, MAX_URL_LENGTH),
        ip_address: payload[:ip_address],
        country_code: payload[:country_code]&.slice(0, 2),
        asn: payload[:asn],
        referrer: payload[:referrer]&.slice(0, MAX_REFERRER_LENGTH),
        user_agent: payload[:user_agent]&.slice(0, MAX_USER_AGENT_LENGTH),
        language: payload[:language]&.slice(0, MAX_LANGUAGE_LENGTH),
        session_id: payload[:session_id]&.slice(0, MAX_SESSION_ID_LENGTH),
        topic_id: payload[:topic_id],
        occurred_at: payload[:occurred_at],
      }
    end

    def deserialize_payload(entry)
      JSON.parse(entry).with_indifferent_access
    end

    def attributes_from_payload(payload)
      normalized_referrer = BrowserPageviewEventUrlNormalizer.normalize_referrer(payload[:referrer])
      normalized_url = BrowserPageviewEventUrlNormalizer.normalize_site_path(payload[:url])
      user_agent = payload[:user_agent]&.slice(0, MAX_USER_AGENT_LENGTH)
      language = payload[:language]&.slice(0, MAX_LANGUAGE_LENGTH)

      {
        url: payload[:url]&.slice(0, MAX_URL_LENGTH),
        normalized_url: normalized_url&.slice(0, MAX_NORMALIZED_URL_LENGTH),
        normalized_url_version: BrowserPageviewEventUrlNormalizer::SITE_PATH_VERSION,
        ip_address: payload[:ip_address],
        country_code: payload[:country_code]&.slice(0, 2),
        asn: payload[:asn],
        referrer: payload[:referrer]&.slice(0, MAX_REFERRER_LENGTH),
        normalized_referrer: normalized_referrer&.slice(0, MAX_NORMALIZED_REFERRER_LENGTH),
        normalized_referrer_version: BrowserPageviewEventUrlNormalizer::REFERRER_VERSION,
        user_agent: user_agent,
        language: language,
        normalized_language: BrowserPageviewEventLanguageNormalizer.normalize(language),
        browser: BROWSERS.fetch(BrowserDetection.browser(user_agent), BROWSER_UNKNOWN),
        session_id: payload[:session_id]&.slice(0, MAX_SESSION_ID_LENGTH),
        user_id: payload[:user_id],
        topic_id: payload[:topic_id],
        created_at: payload[:occurred_at],
      }
    end

    def valid_payload?(payload)
      payload[:url].present? && payload[:ip_address].present? && payload[:user_agent].present? &&
        payload[:session_id].present? && payload[:occurred_at].present? &&
        valid_ip_address?(payload[:ip_address])
    end

    def valid_ip_address?(ip_address)
      IPAddr.new(ip_address)
      true
    rescue IPAddr::Error
      false
    end

    def postgres_connection_error?(error)
      cause = error.cause
      cause.is_a?(PG::ConnectionBad) || cause.is_a?(PG::UnableToSend)
    end
  end

  has_one :browser_pageview_event_score, foreign_key: :event_id, dependent: :delete

  def self.retention_cutoff
    RETENTION_PERIOD.ago.beginning_of_day
  end

  def self.rollup_count_sql
    logged_in_only = SiteSetting.login_required
    count_column = logged_in_only ? "logged_in_count" : "count"
    return count_column if !CrawlerScorer.enabled?

    crawler_column = logged_in_only ? "likely_crawler_logged_in_count" : "likely_crawler_count"
    "GREATEST(#{count_column} - #{crawler_column}, 0)"
  end

  before_save :truncate_fields

  private

  def truncate_fields
    self.url = url.slice(0, MAX_URL_LENGTH) if url.present?
    self.referrer = referrer.slice(0, MAX_REFERRER_LENGTH) if referrer.present?
    self.user_agent = user_agent.slice(0, MAX_USER_AGENT_LENGTH) if user_agent.present?
    self.language = language.slice(0, MAX_LANGUAGE_LENGTH) if language.present?
    if normalized_language.present?
      self.normalized_language = normalized_language.slice(0, MAX_NORMALIZED_LANGUAGE_LENGTH)
    end
    self.session_id = session_id.slice(0, MAX_SESSION_ID_LENGTH) if session_id.present?
    if normalized_referrer.present?
      self.normalized_referrer = normalized_referrer.slice(0, MAX_NORMALIZED_REFERRER_LENGTH)
    end
    if normalized_url.present?
      self.normalized_url = normalized_url.slice(0, MAX_NORMALIZED_URL_LENGTH)
    end
  end
end

# == Schema Information
#
# Table name: browser_pageview_events
#
#  id                          :bigint           not null, primary key
#  asn                         :integer
#  browser                     :integer
#  country_code                :string(2)
#  ip_address                  :inet             not null
#  language                    :string(255)
#  normalized_language         :string
#  normalized_referrer         :string(2000)
#  normalized_referrer_version :integer
#  normalized_url              :string(2000)
#  normalized_url_version      :integer
#  referrer                    :string(2000)
#  score                       :integer
#  url                         :string(2000)     not null
#  user_agent                  :string(1000)     not null
#  created_at                  :datetime         not null
#  session_id                  :string(32)       not null
#  topic_id                    :integer
#  user_id                     :integer
#
# Indexes
#
#  idx_bpe_browser_backfill                     (created_at DESC,id DESC) WHERE (browser IS NULL)
#  idx_bpe_crawler_created_at_covering          (created_at) WHERE (score > 55)
#  idx_bpe_created_at_id                        (created_at DESC,id DESC)
#  idx_bpe_created_at_session_id                (created_at,session_id,source)
#  idx_bpe_ip_created_at                        (ip_address,created_at)
#  idx_bpe_referrer_backfill                    (created_at DESC,id DESC) WHERE ((referrer IS NOT NULL) AND ((normalized_referrer_version IS NULL) OR (normalized_referrer_version < 1)))
#  idx_bpe_session_created_at_covering          (session_id,created_at)
#  idx_bpe_url_backfill                         (created_at DESC,id DESC) WHERE ((normalized_url_version IS NULL) OR (normalized_url_version < 1))
#  index_browser_pageview_events_on_created_at  (created_at) USING brin
#