Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Add 25 default data explorer queries for traffic, health, moderation and audits (#41571)

Rafael dos Santos Silva7b025265d254
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb
index d5860d58be8..44fd1ea138f 100644
--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb
+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb
@@ -116,6 +116,156 @@ module DiscourseDataExplorer
description:
"Details of a Ranked Choice poll result, including details about each vote and voter inc. rank, useful for analyzing results in external software.",
},
+ "weekly-unique-visitors": {
+ id: -20,
+ name: "Weekly Unique Visitors",
+ description:
+ "Number of distinct users who visited the site each week. Accepts a 'weeks_ago' parameter, defaults to the last 12 weeks.",
+ },
+ "top-topics-by-views": {
+ id: -21,
+ name: "Top 100 Topics by Views",
+ description:
+ "The most viewed topics in a recent period, split into anonymous and logged-in views. Accepts a 'days_ago' parameter, defaults to the last 7 days.",
+ },
+ "top-search-terms": {
+ id: -22,
+ name: "Top 200 Search Terms",
+ description:
+ "The most popular search terms by number of distinct users searching for them, useful for spotting content gaps. Accepts a 'days_ago' parameter, defaults to the last 30 days.",
+ },
+ "topic-views-and-clicks": {
+ id: -23,
+ name: "Topic Views and Link Clicks Over Time",
+ description:
+ "Daily views (anonymous and logged in) and outbound link clicks for a single topic, useful for measuring how an announcement performed. Requires a 'topic_id' parameter, accepts a 'days_ago' parameter.",
+ },
+ "avg-first-response-time": {
+ id: -24,
+ name: "Average Time to First Response",
+ description:
+ "Average number of hours before a new topic receives its first reply from someone other than the topic author, grouped by week. Accepts a 'weeks_ago' parameter, defaults to the last 12 weeks.",
+ },
+ "new-topic-response-rate": {
+ id: -25,
+ name: "New Topic Response Rate",
+ description:
+ "Percentage of new public topics that receive any reply, and a staff reply, within 30 days of creation, grouped by month. Only counts topics old enough to have had a fair chance at a reply. Accepts a 'months_ago' parameter.",
+ },
+ "community-participation-trend": {
+ id: -26,
+ name: "Community Participation Trend",
+ description:
+ "Monthly count of distinct non-staff users replying in public topics, with average replies per replier, useful to distinguish 'fewer people participating' from 'the same people posting less'. Accepts a 'months_ago' parameter.",
+ },
+ "trust-level-growth": {
+ id: -27,
+ name: "Trust Level Growth Summary",
+ description:
+ "Number of users who reached each trust level in a recent period, alongside the current total population at each level. Accepts a 'days_ago' parameter, defaults to the last 28 days.",
+ },
+ "topics-with-no-response": {
+ id: -28,
+ name: "Topics With No Response",
+ description:
+ "Number of topics per period that never received a reply from anyone other than the topic author. Accepts 'days_ago', 'category_id', 'include_subcategories' and 'interval' (day, week, month or year) parameters.",
+ },
+ "top-posters": {
+ id: -29,
+ name: "Top Posters in a Given Timeframe",
+ description:
+ "Ranks users by topics created and replies posted in a date range. Requires 'start_date' and 'end_date' parameters (yyyy-mm-dd). Accepts a 'top_x' parameter, defaults to 10.",
+ },
+ "category-activity": {
+ id: -30,
+ name: "Category Activity Breakdown",
+ description:
+ "Topic count, post count, likes and reads per category for a date range. Requires 'start_date' and 'end_date' parameters (yyyy-mm-dd).",
+ },
+ "tl3-promotion-candidates": {
+ id: -31,
+ name: "Trust Level 3 Promotion Progress",
+ description:
+ "Checks every trust level 3 promotion requirement for current trust level 2 users, mirroring the logic of the built-in promotion job, including promotion blockers (locked trust level, active or recent suspensions and silences). Each threshold is a parameter defaulting to the matching site setting's default. Set 'show_all_results' to false to only list users currently meeting every requirement.",
+ },
+ "silenced-users": {
+ id: -32,
+ name: "Silenced Users Report",
+ description:
+ "Currently silenced users, when they were silenced and by whom ('system' for automatic silences). Accepts optional 'start_date', 'end_date' and 'silenced_by' parameters.",
+ },
+ "user-warnings": {
+ id: -33,
+ name: "Recent Official Warnings",
+ description:
+ "Official warnings issued recently, showing who was warned, who issued the warning, and the related topic. Accepts a 'days_ago' parameter, defaults to the last 28 days.",
+ },
+ "most-flagged-users": {
+ id: -34,
+ name: "Users With Most Agreed-Upon Flags",
+ description:
+ "Ranks users by the number of their posts that had a flag agreed with by staff, useful for finding repeat offenders.",
+ },
+ "subcategory-permission-drift": {
+ id: -35,
+ name: "Subcategory Permission Audit",
+ description:
+ "Finds subcategories granting a group permission that the parent category does not grant — a permission-hygiene check not surfaced by the admin UI.",
+ },
+ "reading-participation-histogram": {
+ id: -36,
+ name: "Reading Participation Histogram",
+ description:
+ "Buckets users by how many posts they read in a given period, from a single post up to 2048+, showing the shape of your reading base (lurkers vs power readers). Accepts 'from_days_ago' and 'duration_days' parameters.",
+ },
+ "flags-by-type": {
+ id: -37,
+ name: "Flags by Type",
+ description:
+ "Number of flags per flag type, split into flags reported by real users and flags raised by automated accounts (system, bots). Accepts a 'days_ago' parameter, defaults to the last 90 days.",
+ },
+ "flags-handled-by-staff": {
+ id: -38,
+ name: "Flags Handled by Staff Member",
+ description:
+ "Number of review queue items handled per staff member, useful for recognizing moderation workload. Accepts a 'days_ago' parameter, defaults to the last 90 days.",
+ },
+ "suspended-users": {
+ id: -39,
+ name: "Suspended Users Report",
+ description:
+ "Currently suspended users, when they were suspended, until when, by whom and why. Accepts optional 'start_date', 'end_date' and 'suspended_by' parameters.",
+ },
+ "foreign-language-topics": {
+ id: -40,
+ name: "Topics Not in the Site's Primary Language",
+ description:
+ "Recently created topics whose detected language differs from the site's primary language, useful for routing content for translation or moderation. WARNING: requires content localization with locale detection to be enabled (e.g. the content_localization_enabled and ai_translation_enabled site settings); without it topics have no locale recorded and this report will be empty. Accepts 'primary_locale' and 'days_ago' parameters.",
+ },
+ "foreign-language-posts": {
+ id: -41,
+ name: "Posts Not in the Site's Primary Language",
+ description:
+ "Recent posts whose detected language differs from the site's primary language, with a short excerpt. WARNING: requires content localization with locale detection to be enabled (e.g. the content_localization_enabled and ai_translation_enabled site settings); without it posts have no locale recorded and this report will be empty. Accepts 'primary_locale' and 'days_ago' parameters.",
+ },
+ "crawler-traffic-overview": {
+ id: -42,
+ name: "Crawler and Bot Traffic Overview",
+ description:
+ "Buckets recent pageviews by bot-likelihood score from Discourse's built-in crawler detection, from 'definitely user' to 'definitely crawler'. WARNING: requires browser pageview event collection to be enabled (hidden site setting trigger_browser_pageview_events); without it no events are recorded and this report will be empty. Accepts an 'hours' parameter, defaults to the last 24 hours.",
+ },
+ "crawler-traffic-detailed": {
+ id: -43,
+ name: "Crawler and Bot Traffic Detailed Report",
+ description:
+ "Row-per-IP breakdown of likely bot pageview activity, with the individual signals that drove the score (automated user agent, known crawler network, velocity, session churn, rapid navigation, bad referrer). WARNING: requires browser pageview event collection to be enabled (hidden site setting trigger_browser_pageview_events); without it no events are recorded and this report will be empty. Accepts 'hours' and 'min_score' parameters.",
+ },
+ "suspected-bot-networks": {
+ id: -44,
+ name: "Suspected Automated Traffic by IP and Network",
+ description:
+ "Networks (ASNs) and IPs generating high pageview volume with bot-like session patterns (near 1.0 views per session, rotating user agents, systematic topic harvesting), sorted so a scrape spread across many IPs on one network floats to the top. WARNING: requires browser pageview event collection to be enabled (hidden site setting trigger_browser_pageview_events); without it no events are recorded and this report will be empty. Accepts a 'days_ago' parameter, defaults to the last 3 days.",
+ },
}.with_indifferent_access
queries["most-common-likers"]["sql"] = <<~SQL
@@ -588,6 +738,868 @@ module DiscourseDataExplorer
ORDER BY p.year DESC, qt DESC
SQL
+ queries["weekly-unique-visitors"]["sql"] = <<~SQL
+ -- [params]
+ -- int :weeks_ago = 12
+
+ SELECT
+ DATE_TRUNC('week', visited_at)::date AS week,
+ COUNT(DISTINCT user_id) AS active_users
+ FROM user_visits
+ WHERE visited_at >= CURRENT_DATE - (:weeks_ago * INTERVAL '1 week')
+ GROUP BY week
+ ORDER BY week
+ SQL
+
+ queries["top-topics-by-views"]["sql"] = <<~SQL
+ -- [params]
+ -- int :days_ago = 7
+
+ SELECT
+ t.id AS topic_id,
+ t.category_id,
+ COALESCE(SUM(tvs.anonymous_views), 0) AS anonymous_views,
+ COALESCE(SUM(tvs.logged_in_views), 0) AS logged_in_views,
+ COALESCE(SUM(tvs.anonymous_views + tvs.logged_in_views), 0) AS total_views
+ FROM topics t
+ JOIN topic_view_stats tvs ON tvs.topic_id = t.id
+ AND tvs.viewed_at >= CURRENT_DATE - :days_ago
+ WHERE t.deleted_at IS NULL
+ AND t.archetype = 'regular'
+ GROUP BY t.id, t.category_id
+ ORDER BY total_views DESC
+ LIMIT 100
+ SQL
+
+ queries["top-search-terms"]["sql"] = <<~SQL
+ -- [params]
+ -- int :days_ago = 30
+
+ SELECT
+ term,
+ COUNT(*) AS searches,
+ COUNT(DISTINCT user_id) AS distinct_users
+ FROM search_logs
+ WHERE created_at >= CURRENT_DATE - :days_ago
+ GROUP BY term
+ ORDER BY distinct_users DESC, searches DESC
+ LIMIT 200
+ SQL
+
+ queries["topic-views-and-clicks"]["sql"] = <<~SQL
+ -- [params]
+ -- topic_id :topic_id
+ -- int :days_ago = 30
+
+ -- gapless day series, so days with zero activity still show up as rows
+ WITH day_series AS (
+ SELECT generate_series(
+ CURRENT_DATE - :days_ago,
+ CURRENT_DATE,
+ '1 day'::interval
+ )::date AS day
+ ),
+ daily_clicks AS (
+ SELECT
+ tlc.created_at::date AS day,
+ COUNT(*) AS link_clicks
+ FROM topic_link_clicks tlc
+ JOIN topic_links tl ON tl.id = tlc.topic_link_id
+ WHERE tl.topic_id = :topic_id
+ AND tlc.created_at >= CURRENT_DATE - :days_ago
+ GROUP BY tlc.created_at::date
+ ),
+ daily_views AS (
+ SELECT
+ viewed_at AS day,
+ COALESCE(SUM(anonymous_views), 0) AS anonymous_views,
+ COALESCE(SUM(logged_in_views), 0) AS logged_in_views
+ FROM topic_view_stats
+ WHERE topic_id = :topic_id
+ AND viewed_at >= CURRENT_DATE - :days_ago
+ GROUP BY viewed_at
+ )
+ SELECT
+ ds.day,
+ COALESCE(dv.anonymous_views, 0) + COALESCE(dv.logged_in_views, 0) AS total_views,
+ COALESCE(dv.anonymous_views, 0) AS anonymous_views,
+ COALESCE(dv.logged_in_views, 0) AS logged_in_views,
+ COALESCE(dc.link_clicks, 0) AS link_clicks
+ FROM day_series ds
+ LEFT JOIN daily_views dv ON dv.day = ds.day
+ LEFT JOIN daily_clicks dc ON dc.day = ds.day
+ ORDER BY ds.day
+ SQL
+
+ queries["avg-first-response-time"]["sql"] = <<~SQL
+ -- [params]
+ -- int :weeks_ago = 12
+
+ SELECT
+ week,
+ ROUND(AVG(hours_to_response)::numeric, 1) AS avg_hours_to_first_response
+ FROM (
+ SELECT
+ DATE_TRUNC('week', t.created_at)::date AS week,
+ EXTRACT(EPOCH FROM MIN(p.created_at) - t.created_at) / 3600.0 AS hours_to_response
+ FROM topics t
+ JOIN posts p ON p.topic_id = t.id
+ WHERE t.created_at >= CURRENT_DATE - (:weeks_ago * INTERVAL '1 week')
+ AND t.archetype = 'regular'
+ AND t.deleted_at IS NULL
+ AND p.deleted_at IS NULL
+ AND p.post_number > 1
+ AND p.user_id <> t.user_id
+ AND p.post_type = 1
+ GROUP BY t.id, week
+ ) per_topic
+ GROUP BY week
+ ORDER BY week
+ SQL
+
+ queries["new-topic-response-rate"]["sql"] = <<~SQL
+ -- [params]
+ -- int :months_ago = 6
+
+ WITH base_topics AS (
+ SELECT
+ t.id,
+ t.created_at,
+ date_trunc('month', t.created_at)::date AS month
+ FROM topics t
+ LEFT JOIN categories c ON c.id = t.category_id
+ WHERE t.created_at >= date_trunc('month', CURRENT_DATE) - (:months_ago * INTERVAL '1 month')
+ -- only count topics old enough to have had the full 30-day reply window
+ AND t.created_at < CURRENT_DATE - INTERVAL '30 days'
+ AND t.deleted_at IS NULL
+ AND t.archetype = 'regular'
+ AND COALESCE(c.read_restricted, false) = false
+ AND t.user_id > 0
+ ),
+ topic_responses AS (
+ SELECT
+ bt.month,
+ EXISTS (
+ SELECT 1
+ FROM posts p
+ WHERE p.topic_id = bt.id
+ AND p.post_number > 1
+ AND p.post_type = 1
+ AND p.deleted_at IS NULL
+ AND p.created_at <= bt.created_at + INTERVAL '30 days'
+ ) AS has_any_reply,
+ EXISTS (
+ SELECT 1
+ FROM posts p
+ JOIN users u ON u.id = p.user_id
+ WHERE p.topic_id = bt.id
+ AND p.post_number > 1
+ AND p.post_type = 1
+ AND p.deleted_at IS NULL
+ AND p.created_at <= bt.created_at + INTERVAL '30 days'
+ AND (u.admin OR u.moderator)
+ ) AS has_staff_reply
+ FROM base_topics bt
+ )
+ SELECT
+ month,
+ ROUND(100.0 * COUNT(*) FILTER (WHERE has_any_reply) / COUNT(*), 1) AS any_reply_pct,
+ ROUND(100.0 * COUNT(*) FILTER (WHERE has_staff_reply) / COUNT(*), 1) AS staff_reply_pct
+ FROM topic_responses
+ GROUP BY month
+ ORDER BY month
+ SQL
+
+ queries["community-participation-trend"]["sql"] = <<~SQL
+ -- [params]
+ -- int :months_ago = 6
+
+ WITH monthly AS (
+ SELECT
+ date_trunc('month', p.created_at)::date AS month,
+ COUNT(*) AS replies,
+ COUNT(DISTINCT p.user_id) AS distinct_repliers
+ FROM posts p
+ JOIN topics t ON t.id = p.topic_id
+ JOIN users u ON u.id = p.user_id
+ LEFT JOIN categories c ON c.id = t.category_id
+ WHERE p.created_at >= date_trunc('month', CURRENT_DATE) - (:months_ago * INTERVAL '1 month')
+ AND p.deleted_at IS NULL
+ AND p.post_type = 1
+ AND p.post_number > 1
+ AND t.deleted_at IS NULL
+ AND t.archetype = 'regular'
+ AND COALESCE(c.read_restricted, false) = false
+ AND u.id > 0
+ AND NOT u.admin
+ AND NOT u.moderator
+ GROUP BY 1
+ )
+ SELECT
+ month,
+ distinct_repliers,
+ ROUND(replies::numeric / NULLIF(distinct_repliers, 0), 2) AS replies_per_person
+ FROM monthly
+ ORDER BY month
+ SQL
+
+ queries["trust-level-growth"]["sql"] = <<~SQL
+ -- [params]
+ -- int :days_ago = 28
+
+ -- new signups with no recorded trust level change are still at trust level 0
+ WITH trust_level_0_users AS (
+ SELECT
+ 0 AS trust_level,
+ COUNT(*) AS users_gained
+ FROM users u
+ LEFT JOIN (
+ SELECT DISTINCT target_user_id
+ FROM user_histories
+ WHERE created_at >= CURRENT_DATE - :days_ago
+ AND action IN (2, 15) -- change_trust_level, auto_trust_level_change
+ ) tlc ON u.id = tlc.target_user_id
+ WHERE u.created_at >= CURRENT_DATE - :days_ago
+ AND tlc.target_user_id IS NULL
+ ),
+ trust_level_changes AS (
+ SELECT
+ uh.new_value::int AS trust_level,
+ COUNT(DISTINCT uh.target_user_id) AS users_gained
+ FROM user_histories uh
+ JOIN users u ON uh.target_user_id = u.id
+ WHERE uh.created_at >= CURRENT_DATE - :days_ago
+ AND uh.action IN (2, 15) -- change_trust_level, auto_trust_level_change
+ AND uh.new_value IN ('1', '2', '3', '4')
+ GROUP BY uh.new_value
+ ),
+ trust_levels_combined AS (
+ SELECT generate_series(0, 4) AS trust_level
+ )
+ SELECT
+ t.trust_level,
+ CASE
+ WHEN t.trust_level = 0 THEN COALESCE(tl0.users_gained, 0)
+ ELSE COALESCE(tlc.users_gained, 0)
+ END AS users_gained_recently,
+ (SELECT COUNT(*) FROM users WHERE trust_level = t.trust_level) AS total_users_at_level
+ FROM trust_levels_combined t
+ LEFT JOIN trust_level_changes tlc ON t.trust_level = tlc.trust_level
+ LEFT JOIN trust_level_0_users tl0 ON t.trust_level = 0
+ ORDER BY t.trust_level
+ SQL
+
+ queries["topics-with-no-response"]["sql"] = <<~SQL
+ -- [params]
+ -- int :days_ago = 90
+ -- null category_id :category_id
+ -- boolean :include_subcategories = false
+ -- string :interval = day
+
+ WITH no_response_topics AS (
+ SELECT * FROM (
+ SELECT t.id, t.created_at, MIN(p.post_number) AS first_reply
+ FROM topics t
+ -- reply conditions live in the join, so topics with no replies
+ -- survive with first_reply NULL instead of being filtered out
+ LEFT JOIN posts p ON p.topic_id = t.id
+ AND p.user_id <> t.user_id
+ AND p.deleted_at IS NULL
+ AND p.post_type = 1
+ WHERE t.archetype = 'regular'
+ AND t.deleted_at IS NULL
+ AND t.created_at >= CURRENT_DATE - :days_ago
+ AND (
+ :category_id IS NULL
+ OR t.category_id = :category_id
+ OR (:include_subcategories AND t.category_id IN (
+ SELECT id FROM categories WHERE parent_category_id = :category_id
+ ))
+ )
+ GROUP BY t.id
+ ) tt
+ WHERE tt.first_reply IS NULL OR tt.first_reply < 2
+ )
+ SELECT
+ date_trunc(:interval, created_at)::date AS period,
+ COUNT(id) AS topics_without_response
+ FROM no_response_topics
+ GROUP BY period
+ ORDER BY period
+ SQL
+
+ queries["top-posters"]["sql"] = <<~SQL
+ -- [params]
+ -- date :start_date
+ -- date :end_date
+ -- int :top_x = 10
+
+ SELECT
+ p.user_id,
+ COUNT(*) AS topics_plus_replies,
+ COUNT(*) FILTER (WHERE p.post_number = 1) AS topics,
+ COUNT(*) FILTER (WHERE p.post_number <> 1) AS replies
+ FROM posts p
+ JOIN topics t ON t.id = p.topic_id
+ WHERE p.created_at::date BETWEEN :start_date AND :end_date
+ AND t.archetype = 'regular'
+ AND p.deleted_at IS NULL
+ AND t.deleted_at IS NULL
+ AND p.post_type = 1
+ AND p.user_id > 0
+ GROUP BY p.user_id
+ ORDER BY topics_plus_replies DESC
+ LIMIT :top_x
+ SQL
+
+ queries["category-activity"]["sql"] = <<~SQL
+ -- [params]
+ -- date :start_date
+ -- date :end_date
+
+ SELECT
+ c.id AS category_id,
+ COUNT(DISTINCT t.id) AS topics,
+ COUNT(p.id) AS posts,
+ SUM(p.like_count) AS likes,
+ SUM(p.reads) AS reads
+ FROM categories c
+ JOIN topics t ON t.category_id = c.id
+ JOIN posts p ON p.topic_id = t.id AND p.post_type = 1
+ WHERE p.created_at::date BETWEEN :start_date AND :end_date
+ AND p.deleted_at IS NULL
+ AND t.deleted_at IS NULL
+ GROUP BY c.id
+ ORDER BY COUNT(p.id) DESC
+ SQL
+
+ queries["tl3-promotion-candidates"]["sql"] = <<~SQL
+ -- [params]
+ -- int :tl_time_period = 100
+ -- int :tl_requires_days_visited = 50
+ -- int :tl_requires_topics_replied_to = 10
+ -- int :tl_requires_topics_viewed = 25
+ -- int :tl_requires_topics_viewed_cap = 500
+ -- int :tl_requires_posts_read = 25
+ -- int :tl_requires_posts_read_cap = 20000
+ -- int :tl_requires_max_flagged = 5
+ -- int :tl_requires_topics_viewed_all_time = 200
+ -- int :tl_requires_posts_read_all_time = 500
+ -- int :tl_requires_likes_given = 30
+ -- int :tl_requires_likes_received = 20
+ -- boolean :show_all_results = true
+
+ WITH tl3_candidates AS (
+ SELECT id AS user_id, manual_locked_trust_level, silenced_till, suspended_till
+ FROM users
+ WHERE trust_level = 2
+ AND last_seen_at >= CURRENT_DATE - (:tl_time_period || ' days')::interval
+ ),
+ min_topics_viewed AS (
+ SELECT LEAST(COUNT(*) * (:tl_requires_topics_viewed / 100.0), :tl_requires_topics_viewed_cap) AS min_topics_viewed
+ FROM topics
+ WHERE visible = true AND archetype = 'regular'
+ AND created_at >= CURRENT_DATE - (:tl_time_period || ' days')::interval
+ ),
+ min_posts_read AS (
+ SELECT LEAST(COUNT(*) * (:tl_requires_posts_read / 100.0), :tl_requires_posts_read_cap) AS min_posts_read
+ FROM posts p
+ JOIN topics t ON t.id = p.topic_id
+ WHERE t.deleted_at IS NULL AND t.archetype = 'regular'
+ AND p.deleted_at IS NULL AND p.post_type = 1
+ AND p.created_at >= CURRENT_DATE - (:tl_time_period || ' days')::interval
+ ),
+ min_likes_received_days AS (
+ SELECT LEAST(:tl_requires_likes_received::float / 3.0, 0.75 * :tl_time_period::float)
+ ),
+ days_visited AS (
+ SELECT uv.user_id, COUNT(uv.user_id) AS days_visited
+ FROM user_visits uv
+ JOIN tl3_candidates c ON c.user_id = uv.user_id
+ WHERE visited_at > CURRENT_DATE - (:tl_time_period || ' days')::interval AND posts_read > 0
+ GROUP BY uv.user_id
+ ),
+ num_topics_replied_to AS (
+ SELECT p.user_id, COUNT(DISTINCT p.topic_id) AS topic_reply_count
+ FROM posts p
+ JOIN topics t ON t.id = p.topic_id
+ JOIN tl3_candidates c ON c.user_id = p.user_id
+ WHERE p.user_id <> t.user_id AND t.archetype <> 'private_message'
+ AND p.deleted_at IS NULL AND t.deleted_at IS NULL
+ AND p.created_at >= CURRENT_DATE - (:tl_time_period || ' days')::interval
+ GROUP BY p.user_id
+ ),
+ topics_viewed AS (
+ SELECT tv.user_id, COUNT(tv.user_id) AS topic_view_count
+ FROM topic_views tv
+ JOIN topics t ON t.id = tv.topic_id
+ JOIN tl3_candidates c ON c.user_id = tv.user_id
+ WHERE t.archetype <> 'private_message'
+ AND viewed_at >= CURRENT_DATE - (:tl_time_period || ' days')::interval
+ GROUP BY tv.user_id
+ ),
+ posts_read AS (
+ SELECT uv.user_id, SUM(posts_read) AS posts_read
+ FROM user_visits uv
+ JOIN tl3_candidates c ON c.user_id = uv.user_id
+ WHERE visited_at >= CURRENT_DATE - (:tl_time_period || ' days')::interval
+ GROUP BY uv.user_id
+ ),
+ num_flagged_posts AS (
+ SELECT p.user_id, COUNT(DISTINCT pa.post_id) AS num_flagged_posts
+ FROM post_actions pa
+ JOIN posts p ON p.id = pa.post_id
+ JOIN tl3_candidates c ON c.user_id = p.user_id
+ WHERE p.created_at >= CURRENT_DATE - (:tl_time_period || ' days')::interval
+ AND (spam_count > 0 OR inappropriate_count > 0)
+ AND agreed_at IS NOT NULL AND pa.user_id <> p.user_id
+ GROUP BY p.user_id
+ ),
+ num_flagged_by_users AS (
+ SELECT p.user_id, COUNT(DISTINCT pa.user_id) AS num_flagged_by_users
+ FROM post_actions pa
+ JOIN posts p ON p.id = pa.post_id
+ JOIN tl3_candidates c ON c.user_id = p.user_id
+ WHERE p.created_at >= CURRENT_DATE - (:tl_time_period || ' days')::interval
+ AND (spam_count > 0 OR inappropriate_count > 0)
+ AND agreed_at IS NOT NULL AND pa.user_id <> p.user_id
+ GROUP BY p.user_id
+ ),
+ topics_viewed_all_time AS (
+ SELECT tv.user_id, COUNT(topic_id) AS topics_viewed_all_time
+ FROM topic_views tv
+ JOIN topics t ON t.id = tv.topic_id
+ JOIN tl3_candidates c ON c.user_id = tv.user_id
+ WHERE t.archetype = 'regular'
+ GROUP BY tv.user_id
+ ),
+ posts_read_all_time AS (
+ SELECT uv.user_id, SUM(posts_read) AS posts_read_all_time
+ FROM user_visits uv
+ JOIN tl3_candidates c ON c.user_id = uv.user_id
+ GROUP BY uv.user_id
+ ),
+ num_likes_given AS (
+ SELECT ua.user_id, COUNT(*) AS num_likes_given
+ FROM user_actions ua
+ JOIN topics t ON t.id = ua.target_topic_id
+ JOIN tl3_candidates c ON c.user_id = ua.user_id
+ WHERE ua.created_at >= CURRENT_DATE - (:tl_time_period || ' days')::interval
+ AND t.archetype = 'regular' AND ua.action_type = 1
+ GROUP BY ua.user_id
+ ),
+ num_likes_received AS (
+ SELECT ua.user_id, COUNT(*) AS num_likes_received,
+ COUNT(DISTINCT acting_user_id) AS num_likes_received_users,
+ COUNT(DISTINCT ua.created_at::date) AS num_likes_received_days
+ FROM user_actions ua
+ JOIN topics t ON t.id = ua.target_topic_id
+ JOIN tl3_candidates c ON c.user_id = ua.user_id
+ WHERE ua.created_at >= CURRENT_DATE - (:tl_time_period || ' days')::interval
+ AND t.archetype = 'regular' AND ua.action_type = 2
+ GROUP BY ua.user_id
+ ),
+ penalty_counts AS (
+ -- silences/suspensions in the last 6 months (FORGIVENESS_PERIOD) block promotion,
+ -- but a staff-issued unsilence/unsuspend forgives them (a system-issued one does not)
+ SELECT
+ uh.target_user_id AS user_id,
+ SUM(
+ CASE
+ WHEN uh.action = 30 THEN 1 -- silence_user
+ WHEN uh.action = 31 AND uh.acting_user_id <> -1 THEN -1 -- unsilence_user
+ WHEN uh.action = 10 THEN 1 -- suspend_user
+ WHEN uh.action = 11 AND uh.acting_user_id <> -1 THEN -1 -- unsuspend_user
+ ELSE 0
+ END
+ ) AS penalty_count
+ FROM user_histories uh
+ JOIN tl3_candidates c ON c.user_id = uh.target_user_id
+ WHERE uh.action IN (10, 11, 30, 31)
+ AND uh.created_at > NOW() - INTERVAL '6 months'
+ GROUP BY uh.target_user_id
+ ),
+ candidate_results AS (
+ SELECT
+ c.user_id,
+ c.manual_locked_trust_level IS NULL AS trust_level_unlocked_criteria_met,
+ (c.suspended_till IS NULL OR c.suspended_till <= CURRENT_TIMESTAMP)
+ AND (c.silenced_till IS NULL OR c.silenced_till <= CURRENT_TIMESTAMP) AS not_penalized_criteria_met,
+ COALESCE(pc.penalty_count, 0) AS penalty_count,
+ COALESCE(pc.penalty_count, 0) <= 0 AS penalty_history_criteria_met,
+ COALESCE(days_visited, 0) AS days_visited,
+ COALESCE(days_visited, 0) >= :tl_requires_days_visited AS visits_criteria_met,
+ COALESCE(topic_reply_count, 0) AS topic_reply_count,
+ COALESCE(topic_reply_count, 0) >= :tl_requires_topics_replied_to AS replies_criteria_met,
+ COALESCE(topic_view_count, 0) AS topic_view_count,
+ COALESCE(topic_view_count, 0) >= (SELECT * FROM min_topics_viewed) AS topic_views_criteria_met,
+ COALESCE(posts_read, 0) AS posts_read,
+ COALESCE(posts_read, 0) >= (SELECT * FROM min_posts_read) AS posts_read_criteria_met,
+ COALESCE(num_flagged_posts, 0) AS num_flagged_posts,
+ COALESCE(num_flagged_posts, 0) <= :tl_requires_max_flagged AS flagged_post_criteria_met,
+ COALESCE(num_flagged_by_users, 0) AS num_flagged_by_users,
+ COALESCE(num_flagged_by_users, 0) <= :tl_requires_max_flagged AS flagged_by_users_criteria_met,
+ COALESCE(topics_viewed_all_time, 0) AS topics_viewed_all_time,
+ COALESCE(topics_viewed_all_time, 0) >= :tl_requires_topics_viewed_all_time AS all_time_topic_views_criteria_met,
+ COALESCE(posts_read_all_time, 0) AS posts_read_all_time,
+ COALESCE(posts_read_all_time, 0) >= :tl_requires_posts_read_all_time AS posts_read_all_time_criteria_met,
+ COALESCE(num_likes_given, 0) AS num_likes_given,
+ COALESCE(num_likes_given, 0) >= :tl_requires_likes_given AS likes_given_criteria_met,
+ COALESCE(num_likes_received, 0) AS num_likes_received,
+ COALESCE(num_likes_received, 0) >= :tl_requires_likes_received AS likes_received_criteria_met,
+ COALESCE(num_likes_received_users, 0) AS num_likes_received_users,
+ COALESCE(num_likes_received_users, 0) >= :tl_requires_likes_received::float / 4.0 AS likes_received_users_criteria_met,
+ COALESCE(num_likes_received_days, 0) AS num_likes_received_days,
+ COALESCE(num_likes_received_days, 0) >= (SELECT * FROM min_likes_received_days) AS likes_received_days_criteria_met
+ FROM tl3_candidates c
+ LEFT JOIN days_visited dv ON dv.user_id = c.user_id
+ LEFT JOIN num_topics_replied_to ntr ON ntr.user_id = c.user_id
+ LEFT JOIN topics_viewed tv ON tv.user_id = c.user_id
+ LEFT JOIN posts_read pr ON pr.user_id = c.user_id
+ LEFT JOIN num_flagged_posts nfp ON nfp.user_id = c.user_id
+ LEFT JOIN num_flagged_by_users nfu ON nfu.user_id = c.user_id
+ LEFT JOIN topics_viewed_all_time tvat ON tvat.user_id = c.user_id
+ LEFT JOIN posts_read_all_time prat ON prat.user_id = c.user_id
+ LEFT JOIN num_likes_given nlg ON nlg.user_id = c.user_id
+ LEFT JOIN num_likes_received nlr ON nlr.user_id = c.user_id
+ LEFT JOIN penalty_counts pc ON pc.user_id = c.user_id
+ )
+ SELECT * FROM candidate_results
+ WHERE CASE WHEN :show_all_results THEN true ELSE visits_criteria_met END
+ AND CASE WHEN :show_all_results THEN true ELSE replies_criteria_met END
+ AND CASE WHEN :show_all_results THEN true ELSE topic_views_criteria_met END
+ AND CASE WHEN :show_all_results THEN true ELSE posts_read_criteria_met END
+ AND CASE WHEN :show_all_results THEN true ELSE flagged_post_criteria_met END
+ AND CASE WHEN :show_all_results THEN true ELSE flagged_by_users_criteria_met END
+ AND CASE WHEN :show_all_results THEN true ELSE all_time_topic_views_criteria_met END
+ AND CASE WHEN :show_all_results THEN true ELSE posts_read_all_time_criteria_met END
+ AND CASE WHEN :show_all_results THEN true ELSE likes_given_criteria_met END
+ AND CASE WHEN :show_all_results THEN true ELSE likes_received_criteria_met END
+ AND CASE WHEN :show_all_results THEN true ELSE likes_received_users_criteria_met END
+ AND CASE WHEN :show_all_results THEN true ELSE likes_received_days_criteria_met END
+ AND CASE WHEN :show_all_results THEN true ELSE trust_level_unlocked_criteria_met END
+ AND CASE WHEN :show_all_results THEN true ELSE not_penalized_criteria_met END
+ AND CASE WHEN :show_all_results THEN true ELSE penalty_history_criteria_met END
+ ORDER BY days_visited DESC
+ SQL
+
+ queries["silenced-users"]["sql"] = <<~SQL
+ -- [params]
+ -- null date :start_date
+ -- null date :end_date
+ -- null string :silenced_by
+
+ SELECT
+ silenced_users.id AS user_id,
+ silenced_users.silenced_till AS silenced_till,
+ COALESCE(staff.username, 'system') AS silenced_by,
+ user_histories.created_at AS silenced_at
+ FROM users silenced_users
+ -- lateral join picks only the most recent silence record per user
+ LEFT JOIN LATERAL (
+ SELECT uh.*
+ FROM user_histories uh
+ WHERE uh.target_user_id = silenced_users.id
+ AND uh.action = 30 -- silence_user
+ ORDER BY uh.id DESC
+ LIMIT 1
+ ) user_histories ON true
+ LEFT JOIN users staff
+ ON staff.id = user_histories.acting_user_id
+ WHERE silenced_users.silenced_till > CURRENT_TIMESTAMP
+ AND (:start_date IS NULL OR user_histories.created_at >= :start_date)
+ AND (:end_date IS NULL OR user_histories.created_at <= :end_date)
+ AND (
+ :silenced_by IS NULL
+ OR ((:silenced_by = 'system' AND staff.username IS NULL) OR staff.username = :silenced_by)
+ )
+ ORDER BY user_histories.created_at DESC
+ SQL
+
+ queries["user-warnings"]["sql"] = <<~SQL
+ -- [params]
+ -- int :days_ago = 28
+
+ SELECT
+ w.user_id AS warned_user_id,
+ w.created_by_id AS warning_creator_user_id,
+ w.topic_id,
+ w.created_at
+ FROM user_warnings w
+ WHERE w.created_at >= CURRENT_DATE - :days_ago
+ ORDER BY w.created_at DESC
+ SQL
+
+ queries["most-flagged-users"]["sql"] = <<~SQL
+ SELECT
+ p.user_id,
+ COUNT(DISTINCT pa.post_id) AS flagged_posts,
+ COUNT(*) AS agreed_flags
+ FROM post_actions pa
+ JOIN posts p ON pa.post_id = p.id
+ WHERE pa.agreed_at IS NOT NULL
+ AND p.user_id > 0
+ GROUP BY p.user_id
+ ORDER BY agreed_flags DESC, flagged_posts DESC
+ LIMIT 100
+ SQL
+
+ queries["subcategory-permission-drift"]["sql"] = <<~SQL
+ SELECT subcategories.* FROM (
+ SELECT
+ category.parent_category_id, category.id AS category_id, category.name AS category_name,
+ category_group.permission_type,
+ groups.name AS group_name, groups.id AS group_id
+ FROM categories category
+ INNER JOIN category_groups category_group ON category_group.category_id = category.id
+ INNER JOIN groups ON groups.id = category_group.group_id
+ WHERE parent_category_id IS NOT NULL
+ ) subcategories
+ LEFT JOIN (
+ SELECT
+ category.id AS category_id, category_group.permission_type,
+ groups.id AS group_id
+ FROM categories category
+ INNER JOIN category_groups category_group ON category_group.category_id = category.id
+ INNER JOIN groups ON groups.id = category_group.group_id
+ WHERE parent_category_id IS NULL
+ ) parent_categories
+ ON parent_categories.category_id = subcategories.parent_category_id
+ AND parent_categories.group_id = subcategories.group_id
+ AND parent_categories.permission_type = subcategories.permission_type
+ -- anti-join: keep only grants with no matching grant on the parent category
+ WHERE parent_categories.category_id IS NULL
+ SQL
+
+ queries["reading-participation-histogram"]["sql"] = <<~SQL
+ -- [params]
+ -- int :from_days_ago = 0
+ -- int :duration_days = 28
+
+ WITH t AS (
+ SELECT
+ CURRENT_DATE::timestamp - ((:from_days_ago + :duration_days) * (INTERVAL '1 days')) AS period_start,
+ CURRENT_DATE::timestamp - (:from_days_ago * (INTERVAL '1 days')) AS period_end
+ ),
+ read_visits AS (
+ SELECT user_id, SUM(posts_read) AS posts_read
+ FROM user_visits, t
+ WHERE posts_read >= 1
+ AND visited_at > t.period_start
+ AND visited_at < t.period_end
+ GROUP BY user_id
+ )
+ SELECT
+ CASE
+ WHEN posts_read <= 1 THEN '0001'
+ WHEN posts_read <= 3 THEN '0002 - 0003'
+ WHEN posts_read <= 7 THEN '0004 - 0007'
+ WHEN posts_read <= 15 THEN '0008 - 0015'
+ WHEN posts_read <= 31 THEN '0016 - 0031'
+ WHEN posts_read <= 63 THEN '0032 - 0063'
+ WHEN posts_read <= 127 THEN '0064 - 0127'
+ WHEN posts_read <= 255 THEN '0128 - 0255'
+ WHEN posts_read <= 511 THEN '0256 - 0511'
+ WHEN posts_read <= 1023 THEN '0512 - 1023'
+ WHEN posts_read <= 2047 THEN '1024 - 2047'
+ ELSE '2048+'
+ END AS posts_read_bucket,
+ COUNT(*) AS num_users
+ FROM read_visits
+ GROUP BY posts_read_bucket
+ ORDER BY posts_read_bucket
+ SQL
+
+ queries["flags-by-type"]["sql"] = <<~SQL
+ -- [params]
+ -- int :days_ago = 90
+
+ SELECT
+ COALESCE(f.name, 'score_type_' || rs.reviewable_score_type) AS flag_type,
+ -- system and bot accounts have non-positive user ids
+ COUNT(*) FILTER (WHERE rs.user_id > 0) AS reported_by_users,
+ COUNT(*) FILTER (WHERE rs.user_id <= 0) AS automated,
+ COUNT(*) AS total
+ FROM reviewable_scores rs
+ LEFT JOIN flags f ON f.id = rs.reviewable_score_type
+ WHERE rs.created_at >= CURRENT_DATE - :days_ago
+ GROUP BY 1
+ ORDER BY total DESC
+ SQL
+
+ queries["flags-handled-by-staff"]["sql"] = <<~SQL
+ -- [params]
+ -- int :days_ago = 90
+
+ SELECT
+ rs.reviewed_by_id AS user_id,
+ COUNT(*) AS flags_handled
+ FROM reviewable_scores rs
+ JOIN users u ON u.id = rs.reviewed_by_id
+ WHERE (u.admin OR u.moderator)
+ AND rs.reviewed_at >= CURRENT_DATE - :days_ago
+ GROUP BY rs.reviewed_by_id
+ ORDER BY flags_handled DESC
+ LIMIT 100
+ SQL
+
+ queries["suspended-users"]["sql"] = <<~SQL
+ -- [params]
+ -- null date :start_date
+ -- null date :end_date
+ -- null string :suspended_by
+
+ SELECT
+ u.id AS user_id,
+ u.suspended_at,
+ u.suspended_till,
+ COALESCE(staff.username, 'system') AS suspended_by,
+ uh.created_at AS suspension_logged_at,
+ uh.details
+ FROM users u
+ -- lateral join picks only the most recent suspension record per user
+ LEFT JOIN LATERAL (
+ SELECT *
+ FROM user_histories
+ WHERE target_user_id = u.id
+ AND action = 10 -- suspend_user
+ ORDER BY id DESC
+ LIMIT 1
+ ) uh ON true
+ LEFT JOIN users staff
+ ON staff.id = uh.acting_user_id
+ WHERE u.suspended_till > CURRENT_TIMESTAMP
+ AND (:start_date IS NULL OR uh.created_at >= :start_date)
+ AND (:end_date IS NULL OR uh.created_at <= :end_date)
+ AND (
+ :suspended_by IS NULL
+ OR ((:suspended_by = 'system' AND staff.username IS NULL) OR staff.username = :suspended_by)
+ )
+ ORDER BY uh.created_at DESC NULLS LAST
+ SQL
+
+ queries["foreign-language-topics"]["sql"] = <<~SQL
+ -- [params]
+ -- string :primary_locale = en
+ -- int :days_ago = 30
+
+ SELECT
+ t.locale,
+ t.id AS topic_id,
+ t.category_id,
+ t.created_at
+ FROM topics t
+ WHERE t.locale IS NOT NULL
+ AND t.locale <> ''
+ AND split_part(t.locale, '_', 1) <> split_part(:primary_locale, '_', 1)
+ AND t.created_at >= CURRENT_DATE - :days_ago
+ AND t.deleted_at IS NULL
+ ORDER BY t.created_at DESC
+ SQL
+
+ queries["foreign-language-posts"]["sql"] = <<~SQL
+ -- [params]
+ -- string :primary_locale = en
+ -- int :days_ago = 30
+
+ SELECT
+ p.locale,
+ p.id AS post_id,
+ p.topic_id,
+ LEFT(p.raw, 100) AS excerpt,
+ p.created_at
+ FROM posts p
+ WHERE p.locale IS NOT NULL
+ AND p.locale <> ''
+ AND split_part(p.locale, '_', 1) <> split_part(:primary_locale, '_', 1)
+ AND p.created_at >= CURRENT_DATE - :days_ago
+ AND p.deleted_at IS NULL
+ ORDER BY p.created_at DESC
+ SQL
+
+ queries["crawler-traffic-overview"]["sql"] = <<~SQL
+ -- [params]
+ -- int :hours = 24
+
+ WITH events AS (
+ SELECT COALESCE(score, 0) AS score
+ FROM browser_pageview_events
+ WHERE created_at >= NOW() - (:hours * INTERVAL '1 hour')
+ )
+ SELECT 'Definitely user (0)' AS bucket, COUNT(*) FILTER (WHERE score = 0) AS pageviews FROM events
+ UNION ALL
+ SELECT 'Very likely user (1-40)', COUNT(*) FILTER (WHERE score BETWEEN 1 AND 40) FROM events
+ UNION ALL
+ SELECT 'Maybe crawler (41-99)', COUNT(*) FILTER (WHERE score BETWEEN 41 AND 99) FROM events
+ UNION ALL
+ SELECT 'Definitely crawler (100+)', COUNT(*) FILTER (WHERE score >= 100) FROM events
+ SQL
+
+ queries["crawler-traffic-detailed"]["sql"] = <<~SQL
+ -- [params]
+ -- int :hours = 24
+ -- int :min_score = 40
+
+ SELECT
+ MAX(e.score) AS max_score,
+ e.session_id,
+ e.ip_address,
+ e.user_id,
+ e.user_agent,
+ e.asn,
+ e.country_code,
+ COUNT(*) AS pageviews,
+ MAX(s.automation_ua_score) AS automation_ua,
+ MAX(s.known_asn_score) AS known_asn,
+ MAX(s.velocity_score) AS velocity,
+ MAX(s.churn_score) AS churn,
+ MAX(s.rapid_nav_score) AS rapid_nav,
+ MAX(s.referrer_score) AS referrer,
+ NULLIF(
+ CONCAT_WS(', ',
+ CASE WHEN MAX(s.automation_ua_score) > 0 THEN 'automation UA' END,
+ CASE WHEN MAX(s.known_asn_score) > 0 THEN 'known crawler ASN' END,
+ CASE WHEN MAX(s.velocity_score) > 0 THEN 'high velocity (+' || MAX(s.velocity_score) || ')' END,
+ CASE WHEN MAX(s.churn_score) > 0 THEN 'session churn (+' || MAX(s.churn_score) || ')' END,
+ CASE WHEN MAX(s.rapid_nav_score) > 0 THEN 'rapid navigation' END,
+ CASE WHEN MAX(s.referrer_score) > 0 THEN 'bad referrer (+' || MAX(s.referrer_score) || ')' END
+ ),
+ ''
+ ) AS reasons
+ FROM browser_pageview_events e
+ JOIN browser_pageview_event_scores s ON s.event_id = e.id
+ WHERE e.created_at >= NOW() - (:hours * INTERVAL '1 hour')
+ AND e.score > :min_score
+ GROUP BY e.ip_address, e.user_agent, e.asn, e.country_code, e.user_id, e.session_id
+ ORDER BY max_score DESC, pageviews DESC
+ SQL
+
+ queries["suspected-bot-networks"]["sql"] = <<~SQL
+ -- [params]
+ -- int :days_ago = 3
+
+ SELECT
+ ip_address,
+ asn,
+ country_code,
+ COUNT(*) AS pageviews,
+ COUNT(DISTINCT session_id) AS sessions,
+ ROUND(COUNT(*)::numeric / NULLIF(COUNT(DISTINCT session_id), 0), 1) AS views_per_session,
+ COUNT(DISTINCT user_agent) AS user_agents,
+ COUNT(DISTINCT topic_id) FILTER (WHERE topic_id IS NOT NULL) AS topics_touched,
+ -- network-wide total, so a scrape spread thin across many IPs
+ -- on one network still sorts to the top
+ SUM(COUNT(*)) OVER (PARTITION BY asn) AS asn_total_pageviews,
+ MIN(created_at) AS first_seen,
+ MAX(created_at) AS last_seen,
+ (array_agg(user_agent ORDER BY created_at DESC))[1] AS sample_user_agent
+ FROM browser_pageview_events
+ WHERE created_at >= CURRENT_DATE - :days_ago
+ GROUP BY ip_address, asn, country_code
+ ORDER BY asn_total_pageviews DESC, pageviews DESC
+ LIMIT 100
+ SQL
+
# convert query ids from "mostcommonlikers" to "-1", "mostmessages" to "-2" etc.
queries.transform_keys!.with_index { |key, idx| "-#{idx + 1}" }
queries