A platform for community discussion. Free, open, simple.
# frozen_string_literal: true
class Report
# Change this line each time report format change
# and you want to ensure cache is reset
SCHEMA_VERSION = 5
RELATED_ITEMS_LIMIT = 50
FILTERS = %i[
name
start_date
end_date
category
group
trust_level
file_extension
include_subcategories
hidden_labels
]
MODES = {
table: :table,
chart: :chart,
stacked_chart: :stacked_chart,
stacked_line_chart: :stacked_line_chart,
radar: :radar,
counters: :counters,
inline_table: :inline_table,
storage_stats: :storage_stats,
}
HIDDEN_PAGEVIEW_REPORTS = %w[site_traffic page_view_legacy_total_reqs top_entry_urls]
HIDDEN_LEGACY_PAGEVIEW_REPORTS = %w[
consolidated_page_views_browser_detection
page_view_anon_reqs
page_view_logged_in_reqs
]
ADMIN_ONLY_REPORTS = %w[
admin_logins
top_uploads
topic_view_stats
top_entry_urls
top_referrers_by_browser_pageviews
top_countries_by_browser_pageviews
]
ADMIN_ONLY_RELATED_ITEMS_REPORTS = %w[new_contributors signups]
IP_ADDRESS_REPORTS = %w[suspicious_logins]
def self.hidden?(type, guardian:)
return true if !guardian.is_admin? && ADMIN_ONLY_REPORTS.include?(type)
hidden_reports =
SiteSetting.use_legacy_pageviews ? HIDDEN_PAGEVIEW_REPORTS : HIDDEN_LEGACY_PAGEVIEW_REPORTS
hidden_reports.include?(type)
end
COLORS = {
turquoise: "#1EB8D1",
lime: "#9BC53D",
purple: "#721D8D",
magenta: "#E84A5F",
brown: "#8A6916",
yellow: "#FFCD56",
}
LEGACY_REPORTS = %w[
associated_accounts_by_provider
bookmarks
consolidated_api_requests
flags
flags_status
likes
moderator_warning_private_messages
mobile_visits
notify_moderators_private_messages
notify_user_private_messages
post_edits
profile_views
reactions
suspicious_logins
system_private_messages
top_referrers
top_users_by_likes_received_from_inferior_trust_level
top_users_by_likes_received_from_a_variety_of_people
trust_level_growth
user_flagging_ratio
user_to_user_private_messages
web_hook_events_daily_aggregate
]
INTERNAL_REPORT_TYPES = %w[posters_by_member_type_members]
include Reports::AssociatedAccountsByProvider
include Reports::Bookmarks
include Reports::ConsolidatedApiRequests
include Reports::ConsolidatedPageViews
include Reports::ConsolidatedPageViewsBrowserDetection
include Reports::SiteTraffic
include Reports::DailyEngagedUsers
include Reports::DauByMau
include Reports::Emails
include Reports::Flags
include Reports::FlagsStatus
include Reports::Likes
include Reports::MobileVisits
include Reports::ModeratorWarningPrivateMessages
include Reports::ModeratorsActivity
include Reports::NewContributors
include Reports::NotifyModeratorsPrivateMessages
include Reports::NotifyUserPrivateMessages
include Reports::PostEdits
include Reports::Posts
include Reports::ProfileViews
include Reports::Signups
include Reports::AdminLogins
include Reports::StorageStats
include Reports::SuspiciousLogins
include Reports::SystemPrivateMessages
include Reports::TimeToFirstResponse
include Reports::TopCountriesByBrowserPageviews
include Reports::TopEntryUrls
include Reports::TopIgnoredUsers
include Reports::TopReferredTopics
include Reports::TopReferrers
include Reports::TopReferrersByBrowserPageviews
include Reports::TopTrafficSources
include Reports::TopUploads
include Reports::TopUsersByLikesReceived
include Reports::TopUsersByLikesReceivedFromAVarietyOfPeople
include Reports::TopUsersByLikesReceivedFromInferiorTrustLevel
include Reports::Topics
include Reports::TopicsWithNoResponse
include Reports::TopicViewStats
include Reports::TrendingSearch
include Reports::TrustLevelGrowth
include Reports::TrustLevelPipeline
include Reports::PostersByMemberType
include Reports::ActivityByCategory
include Reports::UserFlaggingRatio
include Reports::UserToUserPrivateMessages
include Reports::UserToUserPrivateMessagesWithReplies
include Reports::UsersByTrustLevel
include Reports::UsersByType
include Reports::Visits
include Reports::WebCrawlers
include Reports::WebHookEventsDailyAggregate
attr_accessor :type,
:data,
:total,
:prev30Days,
:start_date,
:end_date,
:labels,
:prev_period,
:facets,
:limit,
:average,
:percent,
:higher_is_better,
:icon,
:modes,
:prev_data,
:dates_filtering,
:error,
:primary_color,
:secondary_color,
:filters,
:available_filters,
:legacy,
:default_group_by,
:y_axis_title,
:current_user,
:guardian,
:include_related_items,
:related_items,
:related_items_totals
def self.default_days
30
end
def self.default_labels
[
{ type: :date, property: :x, title: I18n.t("reports.default.labels.day") },
{ type: :number, property: :y, title: I18n.t("reports.default.labels.count") },
]
end
def initialize(type)
@type = type
@start_date ||= Report.default_days.days.ago.utc.beginning_of_day
@end_date ||= Time.now.utc.end_of_day
@prev_end_date = @start_date
@average = false
@percent = false
@higher_is_better = true
@modes = [MODES[:chart], MODES[:table]]
@prev_data = nil
@dates_filtering = true
@available_filters = {}
@filters = {}
tertiary = ColorScheme.hex_for_name("tertiary") || "0088cc"
@primary_color = rgba_color(tertiary)
@secondary_color = rgba_color(tertiary, 0.1)
end
def self.cache_key(report)
guardian = report.guardian || report.current_user&.guardian
[
"reports",
report.type,
report.start_date.to_date.strftime("%Y%m%d"),
report.end_date.to_date.strftime("%Y%m%d"),
report.facets,
report.limit,
report.filters.blank? ? nil : MultiJson.dump(report.filters),
SCHEMA_VERSION,
guardian&.user&.id || report.current_user&.id,
guardian&.can_see_ip?,
CrawlerScorer.enabled?,
].compact.map(&:to_s).join(":")
end
def add_filter(name, options = {})
available_filters[name] = options
end
def remove_filter(name)
available_filters.delete(name)
end
def add_category_filter(options = {})
category_id = filters[:category].to_i if filters[:category].present?
add_filter("category", { type: "category", default: category_id }.merge(options))
return if category_id.blank?
include_subcategories = filters[:include_subcategories]
include_subcategories = !!ActiveRecord::Type::Boolean.new.cast(include_subcategories)
add_filter(
"include_subcategories",
{ type: "bool", default: include_subcategories }.merge(options),
)
[category_id, include_subcategories]
end
def self.clear_cache(type = nil)
pattern = type ? "reports:#{type}:*" : "reports:*"
Discourse.cache.keys(pattern).each { |key| Discourse.cache.redis.del(key) }
end
def self.wrap_slow_query(timeout = 20_000)
ActiveRecord::Base.connection.transaction do
# Allows only read only transactions. Skipped in tests, where threads
# share one connection and read-only would break concurrent writes.
DB.exec "SET TRANSACTION READ ONLY" if !Rails.env.test?
# Set a statement timeout so we can't tie up the server
DB.exec "SET LOCAL statement_timeout = #{timeout}"
yield
end
end
def prev_start_date
start_date - (end_date - start_date)
end
def prev_end_date
start_date
end
def as_json(options = nil)
description = I18n.t("reports.#{type}.description", default: "")
description_link = I18n.t("reports.#{type}.description_link", default: "")
{
type: type,
title: I18n.t("reports.#{type}.title", default: nil),
xaxis: I18n.t("reports.#{type}.xaxis", default: nil),
yaxis: I18n.t("reports.#{type}.yaxis", default: nil),
description: description.presence ? description : nil,
description_link: description_link.presence ? description_link : nil,
data: data,
start_date: start_date&.iso8601,
end_date: end_date&.iso8601,
prev_data: prev_data,
prev_start_date: prev_start_date&.iso8601,
prev_end_date: prev_end_date&.iso8601,
prev30Days: prev30Days,
dates_filtering: dates_filtering,
report_key: Report.cache_key(self),
primary_color: primary_color,
secondary_color: secondary_color,
available_filters: available_filters.map { |k, v| { id: k }.merge(v) },
labels: labels || Report.default_labels,
average: average,
percent: percent,
higher_is_better: higher_is_better,
modes: modes,
}.tap do |json|
json[:legacy] = legacy if legacy
json[:icon] = icon if icon
json[:error] = error if error
json[:total] = total if total
json[:prev_period] = prev_period if prev_period
json[:prev30Days] = prev30Days if prev30Days
json[:limit] = limit if limit
json[:default_group_by] = default_group_by if default_group_by
json[:y_axis_title] = y_axis_title if y_axis_title
json[:related_items] = related_items if related_items
json[:related_items_totals] = related_items_totals if related_items_totals
if type == "page_view_crawler_reqs"
json[:related_report] = Report.find(
"web_crawlers",
start_date: start_date,
end_date: end_date,
)&.as_json
end
end
end
def Report.add_report(
name,
exclude_from_dashboard: false,
admin_only_related_items: false,
&block
)
singleton_class.instance_eval { define_method("report_#{name}", &block) }
dashboard_excluded_report_types << name.to_s if exclude_from_dashboard
registered_admin_only_related_items_report_types << name.to_s if admin_only_related_items
end
# Only used for testing.
def Report.remove_report(name)
singleton_class.instance_eval { remove_method("report_#{name}") }
dashboard_excluded_report_types.delete(name.to_s)
registered_admin_only_related_items_report_types.delete(name.to_s)
end
# Report types a plugin has marked as not mountable on the customisable
# admin dashboard. They remain available on the regular reports page.
def Report.dashboard_excluded_report_types
@dashboard_excluded_report_types ||= Set.new
end
def Report.admin_only_related_items_report_types
ADMIN_ONLY_RELATED_ITEMS_REPORTS.to_set | registered_admin_only_related_items_report_types
end
def Report.registered_admin_only_related_items_report_types
@registered_admin_only_related_items_report_types ||= Set.new
end
def self._get(type, opts = nil)
opts ||= {}
type = type.to_s
# Load the report
report = Report.new(type)
report.start_date = opts[:start_date] if opts[:start_date]
report.end_date = opts[:end_date] if opts[:end_date]
report.facets = opts[:facets] || %i[total prev30Days]
report.limit = opts[:limit] if opts[:limit]
report.average = opts[:average] if opts[:average]
report.percent = opts[:percent] if opts[:percent]
report.filters = opts[:filters] if opts[:filters]
report.guardian = opts[:guardian] if opts[:guardian]
report.current_user = opts[:current_user] if opts[:current_user]
report.current_user ||= report.guardian&.user
report.guardian ||= report.current_user&.guardian
report.include_related_items =
opts[:include_related_items] &&
(report.guardian&.is_admin? || !admin_only_related_items_report_types.include?(report.type))
report.labels = Report.default_labels
report.legacy = LEGACY_REPORTS.include?(type)
report
end
def self.find_cached(type, opts = nil)
report = _get(type, opts)
return if report.include_related_items
Discourse.cache.read(cache_key(report))
end
def self.cache(report)
return if report.include_related_items
duration = report.error == :exception ? 1.minute : 60.minutes
Discourse.cache.write(cache_key(report), report.as_json, expires_in: duration)
end
def self.find(type, opts = nil)
opts ||= {}
begin
report = _get(type, opts)
report_method = :"report_#{type}"
begin
wrap_slow_query do
if respond_to?(report_method)
public_send(report_method, report)
elsif type =~ /_reqs\z/
req_report(report, type.split(/_reqs\z/)[0].to_sym)
else
return nil
end
end
rescue ActiveRecord::QueryCanceled, PG::QueryCanceled => e
report.error = :timeout
end
rescue Exception => e
# In test mode, don't swallow exceptions by default to help debug errors.
raise if Rails.env.test? && !opts[:wrap_exceptions_in_test]
# ensures that if anything unexpected prevents us from
# creating a report object we fail elegantly and log an error
if !report
Rails.logger.error("Couldn’t create report `#{type}`: <#{e.class} #{e.message}>")
return nil
end
report.error = :exception
# given reports can be added by plugins we don’t want dashboard failures
# on report computation, however we do want to log which report is provoking
# an error
Rails.logger.error(
"Error while computing report `#{report.type}`: #{e.message}\n#{e.backtrace.join("\n")}",
)
end
report
end
# NOTE: Once use_legacy_pageviews is always false or no longer needed
# we will no longer support the page_view_anon and page_view_logged_in reports,
# they can be removed.
def self.req_report(report, filter = nil)
data =
# For this report we intentionally do not want to count mobile pageviews.
if filter == :page_view_total
SiteSetting.use_legacy_pageviews ? legacy_page_view_requests : page_view_requests
# This is a separate report because if people have switched over
# to _not_ use legacy pageviews, we want to show both a Pageviews
# and Legacy Pageviews report.
elsif filter == :page_view_legacy_total
legacy_page_view_requests
else
ApplicationRequest.where(req_type: ApplicationRequest.req_types[filter])
end
report.icon = "file" if filter == :page_view_total
report.data = []
data
.where("date >= ? AND date <= ?", report.start_date, report.end_date)
.order(date: :asc)
.group(:date)
.sum(:count)
.each { |date, count| report.data << { x: date, y: count } }
report.total = data.sum(:count)
report.prev30Days =
data.where("date >= ? AND date < ?", report.start_date - 31.days, report.start_date).sum(
:count,
)
end
# We purposefully exclude "browser" pageviews. See
# `ConsolidatedPageViewsBrowserDetection` for browser pageviews.
def self.legacy_page_view_requests
ApplicationRequest.where(
req_type: [
ApplicationRequest.req_types[:page_view_crawler],
ApplicationRequest.req_types[:page_view_anon],
ApplicationRequest.req_types[:page_view_logged_in],
].flatten,
)
end
# We purposefully exclude "crawler" pageviews here and by
# only doing browser pageviews we are excluding "other" pageviews
# too. This is to reflect what is shown in the "Site traffic" report
# by default.
def self.page_view_requests
ApplicationRequest.browser_pageviews
end
def self.report_about(report, subject_class, report_method = :count_per_day)
data_start = report.facets.include?(:prev_period) ? report.prev_start_date : report.start_date
counts = subject_class.public_send(report_method, data_start, report.end_date)
prev_period_count = nil
if report.facets.include?(:prev_period)
counts, prev_counts = split_date_counts(counts, report.start_date)
prev_period_count = prev_counts.sum { |_date, count| count }
end
report.data = counts.map { |date, count| { x: date, y: count } }
add_counts report, subject_class, prev_period_count: prev_period_count
end
def self.basic_report_about(report, subject_class, report_method, *args)
report.data = []
subject_class
.public_send(report_method, *args)
.each { |date, count| report.data << { x: date, y: count } }
end
def self.split_date_counts(counts, current_start)
current_start_date = current_start.to_date
counts.partition { |date, _count| date.to_date >= current_start_date }
end
def self.add_prev_data(report, subject_class, report_method, *args)
if report.modes.include?(Report::MODES[:chart]) && report.facets.include?(:prev_period)
prev_data = subject_class.public_send(report_method, *args)
report.prev_data = prev_data.map { |k, v| { x: k, y: v } }
end
end
def self.add_counts(report, subject_class, query_column = "created_at", prev_period_count: nil)
if report.facets.include?(:prev_period)
report.prev_period =
if prev_period_count
prev_period_count
else
subject_class.where(
"#{query_column} >= ? and #{query_column} < ?",
report.prev_start_date,
report.prev_end_date,
).count
end
end
report.total = subject_class.count if report.facets.include?(:total)
if report.facets.include?(:prev30Days)
report.prev30Days =
subject_class.where(
"#{query_column} >= ? and #{query_column} < ?",
report.start_date - 30.days,
report.start_date,
).count
end
end
def self.post_action_report(report, post_action_type)
category_id, include_subcategories = report.add_category_filter
report.data = []
PostAction
.count_per_day_for_type(
post_action_type,
category_id: category_id,
include_subcategories: include_subcategories,
start_date: report.start_date,
end_date: report.end_date,
)
.each { |date, count| report.data << { x: date, y: count } }
countable = PostAction.unscoped.where(post_action_type_id: post_action_type)
if category_id
if include_subcategories
countable =
countable.joins(post: :topic).where(
"topics.category_id IN (?)",
Category.subcategory_ids(category_id),
)
else
countable = countable.joins(post: :topic).where("topics.category_id = ?", category_id)
end
end
add_counts report, countable, "post_actions.created_at"
end
def self.private_messages_report(report, topic_subtype)
report.icon = "envelope"
subject = Topic.where("topics.user_id > 0")
basic_report_about report,
subject,
:private_message_topics_count_per_day,
report.start_date,
report.end_date,
topic_subtype
subject = Topic.private_messages.where("topics.user_id > 0").with_subtype(topic_subtype)
add_counts report, subject, "topics.created_at"
end
def rgba_color(hex, opacity = 1)
rgbs = hex_to_rgbs(adjust_hex(hex))
"rgba(#{rgbs.join(",")},#{opacity})"
end
def colors
COLORS
end
private
def adjust_hex(hex)
hex = hex.gsub("#", "")
if hex.size == 3
chars = hex.scan(/\w/)
hex = chars.zip(chars).flatten.join
end
hex = hex.ljust(6, hex.last) if hex.size < 3
hex
end
def hex_to_rgbs(hex_color)
hex_color = hex_color.gsub("#", "")
rgbs = hex_color.scan(/../)
rgbs.map! { |color| color.hex }.map! { |rgb| rgb.to_i }
end
end