Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Exclude likely crawlers from dashboard metrics (#42611)

Krzysztof Kotlarek08e864298383
diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb
index d173bb6e612..b399e4b68ee 100644
--- a/app/controllers/search_controller.rb
+++ b/app/controllers/search_controller.rb
@@ -54,6 +54,7 @@ class SearchController < ApplicationController
search_args[:search_type] = :full_page
search_args[:ip_address] = request.remote_ip
search_args[:user_agent] = request.user_agent
+ search_args[:session_id] = pageview_session_id
search_args[:user_id] = current_user.id if current_user.present?
if rate_limit_search
@@ -109,6 +110,7 @@ class SearchController < ApplicationController
search_args[:search_type] = :header
search_args[:ip_address] = request.remote_ip
search_args[:user_agent] = request.user_agent
+ search_args[:session_id] = pageview_session_id
search_args[:user_id] = current_user.id if current_user.present?
search_args[:restrict_to_archetype] = params[:restrict_to_archetype] if params[
:restrict_to_archetype
@@ -164,6 +166,10 @@ class SearchController < ApplicationController
protected
+ def pageview_session_id
+ request.headers["HTTP_DISCOURSE_PAGEVIEW_SESSION_ID"]
+ end
+
def site_overloaded?
queue_time = request.env[Middleware::ProcessingRequest::REQUEST_QUEUE_SECONDS_ENV_KEY]
if queue_time
diff --git a/app/jobs/scheduled/detect_crawler_pageviews.rb b/app/jobs/scheduled/detect_crawler_pageviews.rb
index ec76c622999..7c9e2e15129 100644
--- a/app/jobs/scheduled/detect_crawler_pageviews.rb
+++ b/app/jobs/scheduled/detect_crawler_pageviews.rb
@@ -8,10 +8,13 @@ module Jobs
WINDOW_DELAY = BrowserPageviewSessionEngagement::BEACON_SETTLE_PERIOD
def execute(args)
- return if !UpcomingChanges.enabled?(:improved_crawler_detection)
+ return if !CrawlerScorer.enabled?
window_end = Time.now - WINDOW_DELAY
- CrawlerScorer.score!(window_start: window_end - LOOKBACK, window_end: window_end)
+ window_start = window_end - LOOKBACK
+
+ CrawlerScorer.score!(window_start: window_start, window_end: window_end)
+ SearchLog.flag_likely_crawlers!(window_start: window_start, window_end: window_end)
end
end
end
diff --git a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb
index 49ea0395527..14d604d38a4 100644
--- a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb
+++ b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb
@@ -20,7 +20,7 @@ module Jobs
private
def aggregate_crawlers
- return if !UpcomingChanges.enabled?(:improved_crawler_detection)
+ return if !CrawlerScorer.enabled?
start_date, end_date = crawler_aggregation_window
return if start_date.nil?
diff --git a/app/models/browser_pageview_country_daily_rollup.rb b/app/models/browser_pageview_country_daily_rollup.rb
index d7f3e7a29cb..7743fc50dd6 100644
--- a/app/models/browser_pageview_country_daily_rollup.rb
+++ b/app/models/browser_pageview_country_daily_rollup.rb
@@ -6,12 +6,18 @@ class BrowserPageviewCountryDailyRollup < ActiveRecord::Base
end_date = end_date.to_date + 1
DB.exec(<<~SQL, start_date:, end_date:)
- INSERT INTO browser_pageview_country_daily_rollups (date, country_code, count, logged_in_count)
+ INSERT INTO browser_pageview_country_daily_rollups (
+ date, country_code, count, logged_in_count, likely_crawler_count, likely_crawler_logged_in_count
+ )
SELECT
created_at::date AS date,
country_code,
COUNT(*) AS count,
- COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS logged_in_count
+ COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS logged_in_count,
+ COUNT(*) FILTER (WHERE #{CrawlerScorer.likely_crawler_condition}) AS likely_crawler_count,
+ COUNT(*) FILTER (
+ WHERE user_id IS NOT NULL AND #{CrawlerScorer.likely_crawler_condition}
+ ) AS likely_crawler_logged_in_count
FROM browser_pageview_events
WHERE created_at >= :start_date
AND created_at < :end_date
@@ -19,7 +25,9 @@ class BrowserPageviewCountryDailyRollup < ActiveRecord::Base
GROUP BY date, country_code
ON CONFLICT (date, country_code) DO UPDATE
SET count = EXCLUDED.count,
- logged_in_count = EXCLUDED.logged_in_count
+ logged_in_count = EXCLUDED.logged_in_count,
+ likely_crawler_count = EXCLUDED.likely_crawler_count,
+ likely_crawler_logged_in_count = EXCLUDED.likely_crawler_logged_in_count
SQL
end
end
@@ -28,11 +36,13 @@ end
#
# Table name: browser_pageview_country_daily_rollups
#
-# id :bigint not null, primary key
-# count :bigint not null
-# country_code :string(2)
-# date :date not null
-# logged_in_count :bigint not null
+# id :bigint not null, primary key
+# count :bigint not null
+# country_code :string(2)
+# date :date not null
+# likely_crawler_count :bigint default(0), not null
+# likely_crawler_logged_in_count :bigint default(0), not null
+# logged_in_count :bigint not null
#
# Indexes
#
diff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rb
index 2ba7fc58234..75c474b8122 100644
--- a/app/models/browser_pageview_event.rb
+++ b/app/models/browser_pageview_event.rb
@@ -252,6 +252,15 @@ class BrowserPageviewEvent < ActiveRecord::Base
)
end
+ def self.rollup_count_sql
+ logged_in_only = SiteSetting.login_required
+ count_column = logged_in_only ? "logged_in_count" : "count"
+ return count_column if !CrawlerScorer.enabled?
+
+ crawler_column = logged_in_only ? "likely_crawler_logged_in_count" : "likely_crawler_count"
+ "GREATEST(#{count_column} - #{crawler_column}, 0)"
+ end
+
before_save :truncate_fields
private
diff --git a/app/models/browser_pageview_referrer_daily_rollup.rb b/app/models/browser_pageview_referrer_daily_rollup.rb
index 5c8d1e3c51f..a72b6df10c6 100644
--- a/app/models/browser_pageview_referrer_daily_rollup.rb
+++ b/app/models/browser_pageview_referrer_daily_rollup.rb
@@ -6,12 +6,18 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::Base
end_date = end_date.to_date + 1
DB.exec(<<~SQL, start_date:, end_date:)
- INSERT INTO browser_pageview_referrer_daily_rollups (date, normalized_referrer, count, logged_in_count)
+ INSERT INTO browser_pageview_referrer_daily_rollups (
+ date, normalized_referrer, count, logged_in_count, likely_crawler_count, likely_crawler_logged_in_count
+ )
SELECT
created_at::date AS date,
normalized_referrer,
COUNT(*) AS count,
- COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS logged_in_count
+ COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS logged_in_count,
+ COUNT(*) FILTER (WHERE #{CrawlerScorer.likely_crawler_condition}) AS likely_crawler_count,
+ COUNT(*) FILTER (
+ WHERE user_id IS NOT NULL AND #{CrawlerScorer.likely_crawler_condition}
+ ) AS likely_crawler_logged_in_count
FROM browser_pageview_events
WHERE created_at >= :start_date
AND created_at < :end_date
@@ -19,7 +25,9 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::Base
GROUP BY date, normalized_referrer
ON CONFLICT (date, normalized_referrer) DO UPDATE
SET count = EXCLUDED.count,
- logged_in_count = EXCLUDED.logged_in_count
+ logged_in_count = EXCLUDED.logged_in_count,
+ likely_crawler_count = EXCLUDED.likely_crawler_count,
+ likely_crawler_logged_in_count = EXCLUDED.likely_crawler_logged_in_count
SQL
end
@@ -59,11 +67,13 @@ end
#
# Table name: browser_pageview_referrer_daily_rollups
#
-# id :bigint not null, primary key
-# count :bigint not null
-# date :date not null
-# logged_in_count :bigint not null
-# normalized_referrer :string(2000)
+# id :bigint not null, primary key
+# count :bigint not null
+# date :date not null
+# likely_crawler_count :bigint default(0), not null
+# likely_crawler_logged_in_count :bigint default(0), not null
+# logged_in_count :bigint not null
+# normalized_referrer :string(2000)
#
# Indexes
#
diff --git a/app/models/browser_pageview_session_engagement_daily_rollup.rb b/app/models/browser_pageview_session_engagement_daily_rollup.rb
index 78ab6274721..c4b2bf49406 100644
--- a/app/models/browser_pageview_session_engagement_daily_rollup.rb
+++ b/app/models/browser_pageview_session_engagement_daily_rollup.rb
@@ -41,7 +41,8 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Base
bpe.session_id,
MIN(bpe.created_at)::date AS date,
COUNT(*) AS pageview_count,
- bool_or(bpe.user_id IS NOT NULL) AS logged_in
+ bool_or(bpe.user_id IS NOT NULL) AS logged_in,
+ bool_or(#{CrawlerScorer.likely_crawler_condition(table: "bpe")}) AS likely_crawler
FROM browser_pageview_events bpe
JOIN active_sessions ON active_sessions.session_id = bpe.session_id
WHERE #{BrowserPageviewEvent.rollup_source_condition(table: "bpe")}
@@ -49,7 +50,10 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Base
HAVING MIN(bpe.created_at) >= :start_date
)
INSERT INTO browser_pageview_session_engagement_daily_rollups
- (date, logged_in, sessions, bounced, engaged_seconds_total)
+ (
+ date, logged_in, sessions, bounced, engaged_seconds_total,
+ likely_crawler_sessions, likely_crawler_bounced, likely_crawler_engaged_seconds_total
+ )
SELECT
session_pageviews.date,
session_pageviews.logged_in,
@@ -58,7 +62,17 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Base
WHERE session_pageviews.pageview_count = 1
AND COALESCE(engagement.engaged_seconds, 0) < :bounce_threshold
) AS bounced,
- COALESCE(SUM(engagement.engaged_seconds), 0) AS engaged_seconds_total
+ COALESCE(SUM(engagement.engaged_seconds), 0) AS engaged_seconds_total,
+ COUNT(*) FILTER (WHERE session_pageviews.likely_crawler) AS likely_crawler_sessions,
+ COUNT(*) FILTER (
+ WHERE session_pageviews.likely_crawler
+ AND session_pageviews.pageview_count = 1
+ AND COALESCE(engagement.engaged_seconds, 0) < :bounce_threshold
+ ) AS likely_crawler_bounced,
+ COALESCE(
+ SUM(engagement.engaged_seconds) FILTER (WHERE session_pageviews.likely_crawler),
+ 0
+ ) AS likely_crawler_engaged_seconds_total
FROM session_pageviews
LEFT JOIN browser_pageview_session_engagements engagement
ON engagement.session_id = session_pageviews.session_id
@@ -77,12 +91,15 @@ end
#
# Table name: browser_pageview_session_engagement_daily_rollups
#
-# id :bigint not null, primary key
-# bounced :bigint not null
-# date :date not null
-# engaged_seconds_total :bigint not null
-# logged_in :boolean not null
-# sessions :bigint not null
+# id :bigint not null, primary key
+# bounced :bigint not null
+# date :date not null
+# engaged_seconds_total :bigint not null
+# likely_crawler_bounced :bigint default(0), not null
+# likely_crawler_engaged_seconds_total :bigint default(0), not null
+# likely_crawler_sessions :bigint default(0), not null
+# logged_in :boolean not null
+# sessions :bigint not null
#
# Indexes
#
diff --git a/app/models/category_activity_daily_rollup.rb b/app/models/category_activity_daily_rollup.rb
index 8d15f46701f..82536ae1fbe 100644
--- a/app/models/category_activity_daily_rollup.rb
+++ b/app/models/category_activity_daily_rollup.rb
@@ -20,6 +20,8 @@ class CategoryActivityDailyRollup < ActiveRecord::Base
category_ids: nil,
secure_category_ids: nil
)
+ page_views = human_page_views_expr
+
builder = DB.build(<<~SQL)
SELECT
c.id,
@@ -28,15 +30,15 @@ class CategoryActivityDailyRollup < ActiveRecord::Base
c.slug,
COALESCE(SUM(r.topics) FILTER (WHERE r.date >= :current_start), 0)::bigint AS topics_current,
COALESCE(SUM(r.posts) FILTER (WHERE r.date >= :current_start), 0)::bigint AS posts_current,
- COALESCE(SUM(r.page_views) FILTER (WHERE r.date >= :current_start), 0)::bigint AS page_views_current,
+ COALESCE(SUM(#{page_views}) FILTER (WHERE r.date >= :current_start), 0)::bigint AS page_views_current,
COALESCE(SUM(r.topics) FILTER (WHERE r.date < :current_start), 0)::bigint AS topics_prior,
COALESCE(SUM(r.posts) FILTER (WHERE r.date < :current_start), 0)::bigint AS posts_prior,
- COALESCE(SUM(r.page_views) FILTER (WHERE r.date < :current_start), 0)::bigint AS page_views_prior
+ COALESCE(SUM(#{page_views}) FILTER (WHERE r.date < :current_start), 0)::bigint AS page_views_prior
FROM category_activity_daily_rollups r
INNER JOIN categories c ON c.id = r.category_id
/*where*/
GROUP BY c.id, c.name, c.color, c.slug
- HAVING SUM(r.topics + r.posts + r.page_views) > 0
+ HAVING SUM(r.topics + r.posts + #{page_views}) > 0
SQL
builder.where("r.date >= :prev_start AND r.date <= :current_end")
@@ -46,6 +48,13 @@ class CategoryActivityDailyRollup < ActiveRecord::Base
builder.query(prev_start: prev_start, current_start: current_start, current_end: current_end)
end
+ def self.human_page_views_expr
+ return "r.page_views" if !CrawlerScorer.enabled?
+
+ "GREATEST(r.page_views - r.likely_crawler_page_views, 0)"
+ end
+ private_class_method :human_page_views_expr
+
def self.earliest_activity_date
Topic.where(ELIGIBLE_TOPICS).minimum(:created_at)&.to_date
end
@@ -105,12 +114,29 @@ class CategoryActivityDailyRollup < ActiveRecord::Base
AND tvs.viewed_at < :end_date
AND #{ELIGIBLE_TOPICS}
GROUP BY 1, 2
+ ), crawler_view_counts AS (
+ SELECT bpe.created_at::date AS date,
+ topics.category_id,
+ COUNT(DISTINCT (
+ bpe.topic_id,
+ COALESCE(bpe.user_id::text, 'ip-' || bpe.ip_address::text)
+ )) AS likely_crawler_page_views
+ FROM browser_pageview_events bpe
+ INNER JOIN topics ON topics.id = bpe.topic_id
+ WHERE bpe.created_at >= :start_date
+ AND bpe.created_at < :end_date
+ AND #{CrawlerScorer.likely_crawler_condition(table: "bpe")}
+ AND #{BrowserPageviewEvent.rollup_source_condition(table: "bpe")}
+ AND #{ELIGIBLE_TOPICS}
+ GROUP BY 1, 2
)
- SELECT date,
- category_id,
- SUM(topics)::int AS topics,
- SUM(posts)::int AS posts,
- SUM(page_views)::bigint AS page_views
+ SELECT combined.date,
+ combined.category_id,
+ SUM(combined.topics)::int AS topics,
+ SUM(combined.posts)::int AS posts,
+ SUM(combined.page_views)::bigint AS page_views,
+ COALESCE(MAX(crawler_view_counts.likely_crawler_page_views), 0)::bigint
+ AS likely_crawler_page_views
FROM (
SELECT date, category_id, topics, 0 AS posts, 0 AS page_views FROM topic_counts
UNION ALL
@@ -118,7 +144,10 @@ class CategoryActivityDailyRollup < ActiveRecord::Base
UNION ALL
SELECT date, category_id, 0, 0, page_views FROM view_counts
) combined
- GROUP BY date, category_id
+ LEFT JOIN crawler_view_counts
+ ON crawler_view_counts.date = combined.date
+ AND crawler_view_counts.category_id = combined.category_id
+ GROUP BY combined.date, combined.category_id
SQL
start_date: start_date,
end_date: end_date + 1.day,
@@ -128,8 +157,13 @@ class CategoryActivityDailyRollup < ActiveRecord::Base
private_class_method :daily_counts
def self.replace!(start_date:, end_date:, rows:)
+ if rows.empty?
+ where(date: start_date..end_date).delete_all
+ return
+ end
+
+ preserved = crawler_page_views_without_events(start_date, end_date)
where(date: start_date..end_date).delete_all
- return if rows.empty?
insert_all!(
rows.map do |row|
@@ -139,23 +173,45 @@ class CategoryActivityDailyRollup < ActiveRecord::Base
topics: row.topics,
posts: row.posts,
page_views: row.page_views,
+ likely_crawler_page_views:
+ preserved.fetch([row.date, row.category_id], row.likely_crawler_page_views),
}
end,
)
end
private_class_method :replace!
+
+ def self.crawler_page_views_without_events(start_date, end_date)
+ dates_with_events =
+ DB.query_single(<<~SQL, start_date: start_date, end_date: end_date.to_date + 1)
+ SELECT DISTINCT created_at::date
+ FROM browser_pageview_events
+ WHERE created_at >= :start_date
+ AND created_at < :end_date
+ AND #{BrowserPageviewEvent.rollup_source_condition}
+ SQL
+
+ scope = where(date: start_date..end_date).where("likely_crawler_page_views > 0")
+ scope = scope.where.not(date: dates_with_events) if dates_with_events.present?
+
+ scope
+ .pluck(:date, :category_id, :likely_crawler_page_views)
+ .to_h { |date, category_id, count| [[date, category_id], count] }
+ end
+ private_class_method :crawler_page_views_without_events
end
# == Schema Information
#
# Table name: category_activity_daily_rollups
#
-# id :bigint not null, primary key
-# date :date not null
-# page_views :bigint default(0), not null
-# posts :integer default(0), not null
-# topics :integer default(0), not null
-# category_id :integer not null
+# id :bigint not null, primary key
+# date :date not null
+# likely_crawler_page_views :bigint default(0), not null
+# page_views :bigint default(0), not null
+# posts :integer default(0), not null
+# topics :integer default(0), not null
+# category_id :integer not null
#
# Indexes
#
diff --git a/app/models/concerns/reports/site_traffic.rb b/app/models/concerns/reports/site_traffic.rb
index da003046c8d..30519dc9078 100644
--- a/app/models/concerns/reports/site_traffic.rb
+++ b/app/models/concerns/reports/site_traffic.rb
@@ -26,7 +26,7 @@ module Reports::SiteTraffic
page_view_anon_browser: ApplicationRequest.req_types[:page_view_anon_browser],
).first
- likely_crawlers_enabled = UpcomingChanges.enabled?(:improved_crawler_detection)
+ likely_crawlers_enabled = CrawlerScorer.enabled?
data =
DB.query(
diff --git a/app/models/concerns/reports/top_countries_by_browser_pageviews.rb b/app/models/concerns/reports/top_countries_by_browser_pageviews.rb
index c6efe492a99..7b173ffe4dd 100644
--- a/app/models/concerns/reports/top_countries_by_browser_pageviews.rb
+++ b/app/models/concerns/reports/top_countries_by_browser_pageviews.rb
@@ -23,20 +23,20 @@ module Reports::TopCountriesByBrowserPageviews
},
]
- count_expr = SiteSetting.login_required ? "logged_in_count" : "count"
+ count_sql = BrowserPageviewEvent.rollup_count_sql
end_date_exclusive = report.end_date.to_date + 1
sql = <<~SQL
WITH ranked AS (
SELECT
country_code,
- SUM(#{count_expr}) AS count,
- SUM(SUM(#{count_expr})) OVER () AS total
+ SUM(#{count_sql}) AS count,
+ SUM(SUM(#{count_sql})) OVER () AS total
FROM browser_pageview_country_daily_rollups
WHERE date >= :start_date
AND date < :end_date_exclusive
GROUP BY country_code
- HAVING SUM(#{count_expr}) > 0
+ HAVING SUM(#{count_sql}) > 0
)
SELECT country_code, count,
CASE WHEN total = 0 THEN 0
diff --git a/app/models/concerns/reports/top_referrers_by_browser_pageviews.rb b/app/models/concerns/reports/top_referrers_by_browser_pageviews.rb
index 18c4fba4c80..44518e6bc8b 100644
--- a/app/models/concerns/reports/top_referrers_by_browser_pageviews.rb
+++ b/app/models/concerns/reports/top_referrers_by_browser_pageviews.rb
@@ -22,7 +22,7 @@ module Reports::TopReferrersByBrowserPageviews
},
]
- count_expr = SiteSetting.login_required ? "logged_in_count" : "count"
+ count_sql = BrowserPageviewEvent.rollup_count_sql
end_date_exclusive = report.end_date.to_date + 1
host = BrowserPageviewEventUrlNormalizer.normalize_host(Discourse.current_hostname)
@@ -32,8 +32,8 @@ module Reports::TopReferrersByBrowserPageviews
WITH ranked AS (
SELECT
normalized_referrer,
- SUM(#{count_expr}) AS count,
- SUM(SUM(#{count_expr})) OVER () AS total
+ SUM(#{count_sql}) AS count,
+ SUM(SUM(#{count_sql})) OVER () AS total
FROM browser_pageview_referrer_daily_rollups
WHERE date >= :start_date
AND date < :end_date_exclusive
@@ -42,7 +42,7 @@ module Reports::TopReferrersByBrowserPageviews
AND normalized_referrer NOT LIKE :host_path_prefix ESCAPE '\\'
AND normalized_referrer NOT LIKE :host_query_prefix ESCAPE '\\'
GROUP BY normalized_referrer
- HAVING SUM(#{count_expr}) > 0
+ HAVING SUM(#{count_sql}) > 0
)
SELECT normalized_referrer, count,
CASE WHEN total = 0 THEN 0
diff --git a/app/models/report.rb b/app/models/report.rb
index 8c18e1a2c89..4f70e5ded66 100644
--- a/app/models/report.rb
+++ b/app/models/report.rb
@@ -220,6 +220,7 @@ class Report
SCHEMA_VERSION,
guardian&.user&.id || report.current_user&.id,
guardian&.can_see_ip?,
+ CrawlerScorer.enabled?,
].compact.map(&:to_s).join(":")
end
diff --git a/app/models/search_log.rb b/app/models/search_log.rb
index f003a1814cd..e0040597222 100644
--- a/app/models/search_log.rb
+++ b/app/models/search_log.rb
@@ -2,6 +2,7 @@
class SearchLog < ActiveRecord::Base
MAXIMUM_USER_AGENT_LENGTH = 2000
+ MAXIMUM_SESSION_ID_LENGTH = BrowserPageviewEvent::MAX_SESSION_ID_LENGTH
validates :term, presence: true
validates :user_agent, length: { maximum: MAXIMUM_USER_AGENT_LENGTH }
@@ -9,6 +10,15 @@ class SearchLog < ActiveRecord::Base
belongs_to :user
scope :non_staff, -> { joins(:user).where(users: { admin: false, moderator: false }) }
+ scope :non_staff_or_anonymous,
+ -> do
+ left_outer_joins(:user).where(
+ "users.id IS NULL OR (NOT users.admin AND NOT users.moderator)",
+ )
+ end
+ scope :excluding_crawlers, -> { where(crawler: false, likely_crawler: false) }
+ scope :human_only,
+ -> { CrawlerScorer.enabled? ? non_staff_or_anonymous.excluding_crawlers : non_staff }
def ctr
return 0 if click_through == 0 || searches == 0
@@ -37,7 +47,7 @@ class SearchLog < ActiveRecord::Base
Discourse.redis.keys("__SEARCH__LOG_*").each { |k| Discourse.redis.del(k) }
end
- def self.log(term:, search_type:, ip_address:, user_agent: nil, user_id: nil)
+ def self.log(term:, search_type:, ip_address:, user_agent: nil, user_id: nil, session_id: nil)
return [:error] if term.blank?
can_log_search =
@@ -74,6 +84,8 @@ class SearchLog < ActiveRecord::Base
ip_address: ip_address,
user_agent: user_agent,
user_id: user_id,
+ session_id: session_id&.slice(0, MAXIMUM_SESSION_ID_LENGTH),
+ crawler: user_agent.present? && CrawlerDetection.crawler?(user_agent),
)
result = [:created, log.id]
@@ -84,6 +96,47 @@ class SearchLog < ActiveRecord::Base
result
end
+ BACKFILL_AGENT_BATCH_SIZE = 100
+
+ def self.backfill_crawler!
+ agents = DB.query_single(<<~SQL)
+ SELECT DISTINCT user_agent
+ FROM search_logs
+ WHERE NOT crawler
+ AND user_agent IS NOT NULL
+ AND user_agent <> ''
+ SQL
+
+ crawler_agents = agents.select { |agent| CrawlerDetection.crawler?(agent) }
+ return 0 if crawler_agents.empty?
+
+ crawler_agents
+ .each_slice(BACKFILL_AGENT_BATCH_SIZE)
+ .sum { |batch| DB.exec(<<~SQL, agents: batch) }
+ UPDATE search_logs
+ SET crawler = TRUE
+ WHERE NOT crawler
+ AND user_agent IN (:agents)
+ SQL
+ end
+
+ def self.flag_likely_crawlers!(window_start:, window_end:)
+ DB.exec(<<~SQL, window_start: window_start, window_end: window_end)
+ UPDATE search_logs
+ SET likely_crawler = TRUE
+ WHERE NOT search_logs.likely_crawler
+ AND search_logs.session_id IS NOT NULL
+ AND search_logs.created_at >= :window_start
+ AND search_logs.created_at < :window_end
+ AND EXISTS (
+ SELECT 1
+ FROM browser_pageview_events event
+ WHERE event.session_id = search_logs.session_id
+ AND #{CrawlerScorer.likely_crawler_condition(table: "event")}
+ )
+ SQL
+ end
+
def self.term_details(term, period = :weekly, search_type = :all)
details = []
@@ -98,6 +151,7 @@ class SearchLog < ActiveRecord::Base
search_type == :full_page
result = result.where.not(search_result_id: nil) if search_type == :click_through_only
result = result.non_staff if search_type == :non_staff_only
+ result = result.human_only if search_type == :human_only
result
.order("date")
@@ -138,6 +192,8 @@ class SearchLog < ActiveRecord::Base
if search_type == :non_staff_only
result = result.non_staff
+ elsif search_type == :human_only
+ result = result.human_only
elsif search_type != :all
result = result.where("search_type = ?", search_types[search_type])
end
@@ -182,17 +238,21 @@ end
# Table name: search_logs
#
# id :integer not null, primary key
+# crawler :boolean default(FALSE), not null
# ip_address :inet
+# likely_crawler :boolean default(FALSE), not null
# search_result_type :integer
# search_type :integer not null
# term :string not null
# user_agent :string(2000)
# created_at :datetime not null
# search_result_id :integer
+# session_id :string(32)
# user_id :integer
#
# Indexes
#
-# index_search_logs_on_created_at (created_at)
-# index_search_logs_on_user_id_and_created_at (user_id,created_at) WHERE (user_id IS NOT NULL)
+# index_search_logs_on_created_at (created_at)
+# index_search_logs_on_created_at_excluding_crawlers (created_at) WHERE ((NOT crawler) AND (NOT likely_crawler))
+# index_search_logs_on_user_id_and_created_at (user_id,created_at) WHERE (user_id IS NOT NULL)
#
diff --git a/app/services/admin_dashboard_search.rb b/app/services/admin_dashboard_search.rb
index f444e4b5343..86fbab56d90 100644
--- a/app/services/admin_dashboard_search.rb
+++ b/app/services/admin_dashboard_search.rb
@@ -44,6 +44,7 @@ class AdminDashboardSearch
trending: trending,
trending_period: trending_period,
content_gaps: content_gaps,
+ search_type: search_type,
}
end
@@ -100,7 +101,7 @@ class AdminDashboardSearch
def trending
rows =
- non_staff_search_logs_in(window_start: start_date, window_end: end_date)
+ human_search_logs_in(window_start: start_date, window_end: end_date)
.select(<<~SQL)
lower(search_logs.term) AS term,
COUNT(*) AS searches,
@@ -123,7 +124,7 @@ class AdminDashboardSearch
def content_gaps
rows =
- non_staff_search_logs_in(window_start: start_date, window_end: end_date)
+ human_search_logs_in(window_start: start_date, window_end: end_date)
.select(<<~SQL)
lower(search_logs.term) AS term,
COUNT(*) AS searches,
@@ -148,12 +149,12 @@ class AdminDashboardSearch
def window_stats(window_start:, window_end:)
term_stats =
- non_staff_search_logs_in(window_start: window_start, window_end: window_end).select(
- <<~SQL,
+ human_search_logs_in(window_start: window_start, window_end: window_end).select(<<~SQL).group(
COUNT(*) AS searches,
SUM(CASE WHEN search_result_id IS NOT NULL THEN 1 ELSE 0 END) AS clicks
SQL
- ).group("lower(search_logs.term)")
+ "lower(search_logs.term)",
+ )
row = SearchLog.from("(#{term_stats.to_sql}) term_stats").select(<<~SQL).take
COALESCE(SUM(searches), 0)::bigint AS total,
@@ -163,8 +164,12 @@ class AdminDashboardSearch
{ total: row.total, no_match: row.no_match }
end
- def non_staff_search_logs_in(window_start:, window_end:)
- SearchLog.non_staff.where(created_at: window_start..window_end)
+ def human_search_logs_in(window_start:, window_end:)
+ SearchLog.human_only.where(created_at: window_start..window_end)
+ end
+
+ def search_type
+ CrawlerScorer.enabled? ? "human_only" : "non_staff_only"
end
def parse_date(value)
diff --git a/app/services/admin_dashboard_site_traffic.rb b/app/services/admin_dashboard_site_traffic.rb
index 04b34a0a19b..cb5bb654e87 100644
--- a/app/services/admin_dashboard_site_traffic.rb
+++ b/app/services/admin_dashboard_site_traffic.rb
@@ -105,7 +105,7 @@ class AdminDashboardSiteTraffic
def likely_crawlers_enabled?
return @likely_crawlers_enabled if defined?(@likely_crawlers_enabled)
- @likely_crawlers_enabled = UpcomingChanges.enabled?(:improved_crawler_detection)
+ @likely_crawlers_enabled = CrawlerScorer.enabled?
end
def kpis(totals, prior_rows)
@@ -144,9 +144,12 @@ class AdminDashboardSiteTraffic
DB
.query_hash(<<~SQL, start_date: start_date.to_date, end_date: end_date.to_date)
SELECT
- COALESCE(SUM(sessions), 0)::bigint AS sessions,
- COALESCE(SUM(bounced), 0)::bigint AS bounced,
- COALESCE(SUM(engaged_seconds_total), 0)::bigint AS engaged_seconds_total
+ COALESCE(SUM(#{human_column("sessions", "likely_crawler_sessions")}), 0)::bigint AS sessions,
+ COALESCE(SUM(#{human_column("bounced", "likely_crawler_bounced")}), 0)::bigint AS bounced,
+ COALESCE(
+ SUM(#{human_column("engaged_seconds_total", "likely_crawler_engaged_seconds_total")}),
+ 0
+ )::bigint AS engaged_seconds_total
FROM browser_pageview_session_engagement_daily_rollups
WHERE date >= :start_date
AND date <= :end_date
@@ -155,6 +158,12 @@ class AdminDashboardSiteTraffic
.symbolize_keys
end
+ def human_column(column, crawler_column)
+ return column if !likely_crawlers_enabled?
+
+ "GREATEST(#{column} - #{crawler_column}, 0)"
+ end
+
def browser_pageviews_kpi(totals, prior_rows)
kpi = { value: totals[:human] }
trend = build_trend(totals, prior_rows)
@@ -173,12 +182,12 @@ class AdminDashboardSiteTraffic
def direct_traffic_value
return nil if !SiteSetting.persist_browser_pageview_events
- count_column = login_required? ? "logged_in_count" : "count"
+ count_sql = BrowserPageviewEvent.rollup_count_sql
row = DB.query(<<~SQL, start_date: start_date.to_date, end_date: end_date.to_date).first
SELECT
- COALESCE(SUM(#{count_column}), 0)::bigint AS total,
- COALESCE(SUM(#{count_column}) FILTER (WHERE normalized_referrer IS NULL), 0)::bigint AS direct
+ COALESCE(SUM(#{count_sql}), 0)::bigint AS total,
+ COALESCE(SUM(#{count_sql}) FILTER (WHERE normalized_referrer IS NULL), 0)::bigint AS direct
FROM browser_pageview_referrer_daily_rollups
WHERE date >= :start_date
AND date <= :end_date
diff --git a/app/services/admin_dashboard_site_traffic_explorer.rb b/app/services/admin_dashboard_site_traffic_explorer.rb
index 699b5a89649..47c79f533e6 100644
--- a/app/services/admin_dashboard_site_traffic_explorer.rb
+++ b/app/services/admin_dashboard_site_traffic_explorer.rb
@@ -155,7 +155,7 @@ class AdminDashboardSiteTrafficExplorer
include_logged_in: filters[:traffic_type]&.include?("logged_in"),
include_anonymous: filters[:traffic_type]&.include?("anonymous"),
include_likely_crawler: filters[:traffic_type]&.include?("likely_crawler"),
- crawler_detection_enabled: UpcomingChanges.enabled?(:improved_crawler_detection),
+ crawler_detection_enabled: CrawlerScorer.enabled?,
crawler_threshold: CrawlerScorer::BOT_SCORE_THRESHOLD,
bounce_threshold:
BrowserPageviewSessionEngagementDailyRollup.bounce_engaged_seconds_threshold,
diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml
index 0551883db07..772af7d26af 100644
--- a/config/locales/client.en.yml
+++ b/config/locales/client.en.yml
@@ -7574,7 +7574,7 @@ en:
kpi:
total_searches:
label: "Total searches"
- tooltip: "The number of searches performed by members in your community."
+ tooltip: "The number of searches performed in your community."
no_result_rate:
label: "No-result rate"
tooltip: "The percentage of searches where members didn't click any result (0% CTR). This indicates that members may be searching for content your community doesn't have."
@@ -7584,7 +7584,7 @@ en:
status: "Status"
trending:
title: "Trending searches"
- tooltip: "The most popular search terms among members."
+ tooltip: "The most popular search terms in your community."
empty: "No searches in this period."
content_gaps:
title: "Content gaps"
@@ -9223,6 +9223,7 @@ en:
full_page: "Full page"
click_through_only: "All (click through only)"
non_staff_only: "Non-staff users"
+ human_only: "Excluding staff and crawlers"
header_search_results: "Header search results"
logster:
title: "Error logs"
diff --git a/db/migrate/20260813071223_add_likely_crawler_columns.rb b/db/migrate/20260813071223_add_likely_crawler_columns.rb
new file mode 100644
index 00000000000..d0e4803bb8e
--- /dev/null
+++ b/db/migrate/20260813071223_add_likely_crawler_columns.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class AddLikelyCrawlerColumns < ActiveRecord::Migration[8.0]
+ def change
+ add_column :browser_pageview_country_daily_rollups,
+ :likely_crawler_count,
+ :bigint,
+ null: false,
+ default: 0
+ add_column :browser_pageview_country_daily_rollups,
+ :likely_crawler_logged_in_count,
+ :bigint,
+ null: false,
+ default: 0
+
+ add_column :browser_pageview_referrer_daily_rollups,
+ :likely_crawler_count,
+ :bigint,
+ null: false,
+ default: 0
+ add_column :browser_pageview_referrer_daily_rollups,
+ :likely_crawler_logged_in_count,
+ :bigint,
+ null: false,
+ default: 0
+
+ add_column :browser_pageview_session_engagement_daily_rollups,
+ :likely_crawler_sessions,
+ :bigint,
+ null: false,
+ default: 0
+ add_column :browser_pageview_session_engagement_daily_rollups,
+ :likely_crawler_bounced,
+ :bigint,
+ null: false,
+ default: 0
+ add_column :browser_pageview_session_engagement_daily_rollups,
+ :likely_crawler_engaged_seconds_total,
+ :bigint,
+ null: false,
+ default: 0
+
+ add_column :category_activity_daily_rollups,
+ :likely_crawler_page_views,
+ :bigint,
+ null: false,
+ default: 0
+
+ add_column :search_logs, :crawler, :boolean, null: false, default: false
+ add_column :search_logs, :likely_crawler, :boolean, null: false, default: false
+ add_column :search_logs, :session_id, :string, limit: 32
+ end
+end
diff --git a/db/migrate/20260813071230_add_search_logs_created_at_excluding_crawlers_index.rb b/db/migrate/20260813071230_add_search_logs_created_at_excluding_crawlers_index.rb
new file mode 100644
index 00000000000..6679f334a9c
--- /dev/null
+++ b/db/migrate/20260813071230_add_search_logs_created_at_excluding_crawlers_index.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class AddSearchLogsCreatedAtExcludingCrawlersIndex < ActiveRecord::Migration[8.0]
+ disable_ddl_transaction!
+
+ INDEX_NAME = "index_search_logs_on_created_at_excluding_crawlers"
+
+ def up
+ remove_index :search_logs, name: INDEX_NAME, algorithm: :concurrently, if_exists: true
+ add_index :search_logs,
+ :created_at,
+ name: INDEX_NAME,
+ where: "NOT crawler AND NOT likely_crawler",
+ algorithm: :concurrently
+ end
+
+ def down
+ remove_index :search_logs, name: INDEX_NAME, algorithm: :concurrently, if_exists: true
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index 1eb5810955e..290c6592911 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1957,7 +1957,9 @@ CREATE TABLE public.browser_pageview_country_daily_rollups (
date date NOT NULL,
country_code character varying(2),
count bigint NOT NULL,
- logged_in_count bigint NOT NULL
+ logged_in_count bigint NOT NULL,
+ likely_crawler_count bigint DEFAULT 0 NOT NULL,
+ likely_crawler_logged_in_count bigint DEFAULT 0 NOT NULL
);
@@ -2105,7 +2107,9 @@ CREATE TABLE public.browser_pageview_referrer_daily_rollups (
date date NOT NULL,
normalized_referrer character varying(2000),
count bigint NOT NULL,
- logged_in_count bigint NOT NULL
+ logged_in_count bigint NOT NULL,
+ likely_crawler_count bigint DEFAULT 0 NOT NULL,
+ likely_crawler_logged_in_count bigint DEFAULT 0 NOT NULL
);
@@ -2138,7 +2142,10 @@ CREATE TABLE public.browser_pageview_session_engagement_daily_rollups (
logged_in boolean NOT NULL,
sessions bigint NOT NULL,
bounced bigint NOT NULL,
- engaged_seconds_total bigint NOT NULL
+ engaged_seconds_total bigint NOT NULL,
+ likely_crawler_sessions bigint DEFAULT 0 NOT NULL,
+ likely_crawler_bounced bigint DEFAULT 0 NOT NULL,
+ likely_crawler_engaged_seconds_total bigint DEFAULT 0 NOT NULL
);
@@ -2281,7 +2288,8 @@ CREATE TABLE public.category_activity_daily_rollups (
category_id integer NOT NULL,
topics integer DEFAULT 0 NOT NULL,
posts integer DEFAULT 0 NOT NULL,
- page_views bigint DEFAULT 0 NOT NULL
+ page_views bigint DEFAULT 0 NOT NULL,
+ likely_crawler_page_views bigint DEFAULT 0 NOT NULL
);
@@ -8803,7 +8811,10 @@ CREATE TABLE public.search_logs (
search_type integer NOT NULL,
created_at timestamp without time zone NOT NULL,
search_result_type integer,
- user_agent character varying(2000)
+ user_agent character varying(2000),
+ crawler boolean DEFAULT false NOT NULL,
+ likely_crawler boolean DEFAULT false NOT NULL,
+ session_id character varying(32)
);
@@ -21273,6 +21284,13 @@ CREATE UNIQUE INDEX index_screened_urls_on_url ON public.screened_urls USING btr
CREATE INDEX index_search_logs_on_created_at ON public.search_logs USING btree (created_at);
+--
+-- Name: index_search_logs_on_created_at_excluding_crawlers; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_search_logs_on_created_at_excluding_crawlers ON public.search_logs USING btree (created_at) WHERE ((NOT crawler) AND (NOT likely_crawler));
+
+
--
-- Name: index_search_logs_on_user_id_and_created_at; Type: INDEX; Schema: public; Owner: -
--
@@ -23178,6 +23196,8 @@ INSERT INTO "schema_migrations" (version) VALUES
('20260814060100'),
('20260814060000'),
('20260813202233'),
+('20260813071230'),
+('20260813071223'),
('20260812094609'),
('20260811231259'),
('20260810154331'),
diff --git a/frontend/discourse/admin/components/dashboard/search.gjs b/frontend/discourse/admin/components/dashboard/search.gjs
index a205057fcf7..d45a46873df 100644
--- a/frontend/discourse/admin/components/dashboard/search.gjs
+++ b/frontend/discourse/admin/components/dashboard/search.gjs
@@ -273,7 +273,7 @@ export default class DashboardSearch extends Component {
@query={{hash
term=row.term
period=this.trendingTermPeriod
- searchType="non_staff_only"
}}
title={{row.term}}
>
diff --git a/frontend/discourse/admin/controllers/admin-search-logs/index.js b/frontend/discourse/admin/controllers/admin-search-logs/index.js
index 8cbb815ee2d..c1eccaa7eb8 100644
--- a/frontend/discourse/admin/controllers/admin-search-logs/index.js
+++ b/frontend/discourse/admin/controllers/admin-search-logs/index.js
@@ -1,25 +1,39 @@
import Controller from "@ember/controller";
+import { service } from "@ember/service";
import { i18n } from "discourse-i18n";
export const DEFAULT_PERIOD = "yearly";
export default class AdminSearchLogsIndexController extends Controller {
+ @service siteSettings;
+
loading = false;
period = DEFAULT_PERIOD;
searchType = "all";
- searchTypeOptions = [
- {
- id: "all",
- name: i18n("admin.logs.search_logs.types.all_search_types"),
- },
- {
- id: "non_staff_only",
- name: i18n("admin.logs.search_logs.types.non_staff_only"),
- },
- { id: "header", name: i18n("admin.logs.search_logs.types.header") },
- {
- id: "full_page",
- name: i18n("admin.logs.search_logs.types.full_page"),
- },
- ];
+
+ get searchTypeOptions() {
+ return [
+ {
+ id: "all",
+ name: i18n("admin.logs.search_logs.types.all_search_types"),
+ },
+ {
+ id: "non_staff_only",
+ name: i18n("admin.logs.search_logs.types.non_staff_only"),
+ },
+ ...(this.siteSettings.improved_crawler_detection
+ ? [
+ {
+ id: "human_only",
+ name: i18n("admin.logs.search_logs.types.human_only"),
+ },
+ ]
+ : []),
+ { id: "header", name: i18n("admin.logs.search_logs.types.header") },
+ {
+ id: "full_page",
+ name: i18n("admin.logs.search_logs.types.full_page"),
+ },
+ ];
+ }
}
diff --git a/frontend/discourse/admin/controllers/admin-search-logs/term.js b/frontend/discourse/admin/controllers/admin-search-logs/term.js
index c37c4d0c2d1..4958044a60f 100644
--- a/frontend/discourse/admin/controllers/admin-search-logs/term.js
+++ b/frontend/discourse/admin/controllers/admin-search-logs/term.js
@@ -1,31 +1,45 @@
import Controller from "@ember/controller";
+import { service } from "@ember/service";
import { DEFAULT_PERIOD } from "discourse/admin/controllers/admin-search-logs/index";
import { i18n } from "discourse-i18n";
export default class AdminSearchLogsTermController extends Controller {
+ @service siteSettings;
+
loading = false;
term = null;
period = DEFAULT_PERIOD;
searchType = "all";
- searchTypeOptions = [
- {
- id: "all",
- name: i18n("admin.logs.search_logs.types.all_search_types"),
- },
- {
- id: "non_staff_only",
- name: i18n("admin.logs.search_logs.types.non_staff_only"),
- },
- { id: "header", name: i18n("admin.logs.search_logs.types.header") },
- {
- id: "full_page",
- name: i18n("admin.logs.search_logs.types.full_page"),
- },
- {
- id: "click_through_only",
- name: i18n("admin.logs.search_logs.types.click_through_only"),
- },
- ];
+
+ get searchTypeOptions() {
+ return [
+ {
+ id: "all",
+ name: i18n("admin.logs.search_logs.types.all_search_types"),
+ },
+ {
+ id: "non_staff_only",
+ name: i18n("admin.logs.search_logs.types.non_staff_only"),
+ },
+ ...(this.siteSettings.improved_crawler_detection
+ ? [
+ {
+ id: "human_only",
+ name: i18n("admin.logs.search_logs.types.human_only"),
+ },
+ ]
+ : []),
+ { id: "header", name: i18n("admin.logs.search_logs.types.header") },
+ {
+ id: "full_page",
+ name: i18n("admin.logs.search_logs.types.full_page"),
+ },
+ {
+ id: "click_through_only",
+ name: i18n("admin.logs.search_logs.types.click_through_only"),
+ },
+ ];
+ }
get chartConfig() {
return {
diff --git a/frontend/discourse/app/controllers/full-page-search.js b/frontend/discourse/app/controllers/full-page-search.js
index 4f36cfacde9..bfc3011aed4 100644
--- a/frontend/discourse/app/controllers/full-page-search.js
+++ b/frontend/discourse/app/controllers/full-page-search.js
@@ -479,7 +479,16 @@ export default class FullPageSearchController extends Controller {
if (this.currentUser) {
updateRecentSearches(this.currentUser, searchTerm);
}
- ajax("/search", { data: args })
+ const sessionId = document.querySelector(
+ "meta[name=discourse-track-view-session-id]"
+ )?.content;
+
+ ajax("/search", {
+ data: args,
+ headers: sessionId
+ ? { "Discourse-Pageview-Session-Id": sessionId }
+ : {},
+ })
.then(async (results) => {
const model = (await translateResults(results)) || {};
diff --git a/frontend/discourse/app/lib/search.js b/frontend/discourse/app/lib/search.js
index 27747ae5fbd..07499241cd6 100644
--- a/frontend/discourse/app/lib/search.js
+++ b/frontend/discourse/app/lib/search.js
@@ -189,7 +189,14 @@ export function searchForTerm(term, opts) {
};
}
- let ajaxPromise = ajax("/search/query", { data });
+ const sessionId = document.querySelector(
+ "meta[name=discourse-track-view-session-id]"
+ )?.content;
+ const headers = sessionId
+ ? { "Discourse-Pageview-Session-Id": sessionId }
+ : {};
+
+ let ajaxPromise = ajax("/search/query", { data, headers });
const promise = ajaxPromise.then((res) => translateResults(res, opts));
promise.abort = ajaxPromise.abort;
return promise;
diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rb
index 2cc0dc0165f..1fb50682ec0 100644
--- a/lib/crawler_scorer.rb
+++ b/lib/crawler_scorer.rb
@@ -44,6 +44,15 @@ class CrawlerScorer
MISSING_ENGAGEMENT_HIGH_SCORE = 40
ENGAGEMENT_LOOKBACK = 6.hours
+ def self.enabled?
+ UpcomingChanges.enabled?(:improved_crawler_detection)
+ end
+
+ def self.likely_crawler_condition(table: nil)
+ prefix = table ? "#{table}." : ""
+ "#{prefix}score > #{BOT_SCORE_THRESHOLD}"
+ end
+
def self.score!(window_start:, window_end:)
crawler_asns = SiteSetting.crawler_asns_map.map(&:to_i)
crawler_detection_datacenter_asns =
diff --git a/lib/dashboard_rollup_rebuilder.rb b/lib/dashboard_rollup_rebuilder.rb
new file mode 100644
index 00000000000..1221568871b
--- /dev/null
+++ b/lib/dashboard_rollup_rebuilder.rb
@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+# Recomputes the rollup tables behind the admin dashboard from their source
+# records. Replaying never drops data that can no longer be derived, because
+# browser pageview events are pruned long before the rollups are. The flip side
+# is that rows whose source events are gone are refreshed in place rather than
+# removed, so a replay cannot retract counts that were once written.
+class DashboardRollupRebuilder
+ NAMES = %w[
+ browser_pageview_country
+ browser_pageview_referrer
+ browser_pageview_session_engagement
+ browser_pageview_crawler
+ category_activity
+ user_visit
+ ].freeze
+
+ def self.names
+ NAMES
+ end
+
+ def self.known?(name)
+ NAMES.include?(name.to_s)
+ end
+
+ def self.rebuild!(name = nil, io: $stdout)
+ new(io: io).rebuild!(name)
+ end
+
+ def initialize(io: $stdout)
+ @io = io
+ end
+
+ def rebuild!(name = nil)
+ rollups = name ? [name.to_s] : NAMES
+
+ rollups.each do |rollup|
+ log "Rebuilding #{rollup} rollups for #{RailsMultisite::ConnectionManagement.current_db}"
+ public_send(:"rebuild_#{rollup}")
+ end
+ end
+
+ def rebuild_browser_pageview_country
+ aggregate_browser_pageviews(BrowserPageviewCountryDailyRollup)
+ end
+
+ def rebuild_browser_pageview_referrer
+ aggregate_browser_pageviews(BrowserPageviewReferrerDailyRollup)
+ end
+
+ def rebuild_browser_pageview_session_engagement
+ aggregate_browser_pageviews(BrowserPageviewSessionEngagementDailyRollup)
+ end
+
+ def rebuild_browser_pageview_crawler
+ aggregate_browser_pageviews(BrowserPageviewCrawlerDailyRollup)
+ end
+
+ def rebuild_category_activity
+ CategoryActivityDailyRollup.rebuild!
+ end
+
+ def rebuild_user_visit
+ start_date = UserVisit.minimum(:visited_at)&.to_date
+ return log(" no user visits, skipping") if start_date.nil?
+
+ UserVisitDailyRollup.aggregate(start_date: start_date, end_date: Time.zone.today)
+ end
+
+ private
+
+ def aggregate_browser_pageviews(rollup)
+ start_date = earliest_event_date
+ return log(" no browser pageview events, skipping") if start_date.nil?
+
+ rollup.aggregate(start_date: start_date, end_date: Time.zone.today)
+ end
+
+ def earliest_event_date
+ @earliest_event_date ||=
+ BrowserPageviewEvent
+ .where(BrowserPageviewEvent.rollup_source_condition)
+ .minimum(:created_at)
+ &.to_date
+ end
+
+ def log(message)
+ @io.puts(message)
+ end
+end
diff --git a/lib/search.rb b/lib/search.rb
index 963e3e1a0d1..f18116abdb0 100644
--- a/lib/search.rb
+++ b/lib/search.rb
@@ -336,6 +336,7 @@ class Search
ip_address: @opts[:ip_address],
user_agent: @opts[:user_agent],
user_id: @opts[:user_id],
+ session_id: @opts[:session_id],
)
@results.search_log_id = search_log_id unless status == :error
end
diff --git a/lib/tasks/admin_dashboard.rake b/lib/tasks/admin_dashboard.rake
new file mode 100644
index 00000000000..e8403822ebb
--- /dev/null
+++ b/lib/tasks/admin_dashboard.rake
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+def each_dashboard_db(&block)
+ if ENV["RAILS_DB"]
+ block.call
+ else
+ RailsMultisite::ConnectionManagement.each_connection(&block)
+ end
+end
+
+desc "Rebuild admin dashboard rollup tables. Pass a name to rebuild just one."
+task "admin_dashboard:rebuild_rollups", [:name] => :environment do |_, args|
+ name = args[:name].presence
+
+ if name && !DashboardRollupRebuilder.known?(name)
+ abort "Unknown rollup '#{name}'. Known rollups: #{DashboardRollupRebuilder.names.join(", ")}"
+ end
+
+ each_dashboard_db { DashboardRollupRebuilder.rebuild!(name) }
+end
+
+desc "Classify existing search logs from known crawler user agents as crawler traffic"
+task "admin_dashboard:backfill_search_log_crawlers" => :environment do
+ each_dashboard_db do
+ db = RailsMultisite::ConnectionManagement.current_db
+ flagged = SearchLog.backfill_crawler!
+ puts "#{db}: flagged #{flagged} search logs from known crawler user agents"
+
+ next if !CrawlerScorer.enabled?
+
+ likely =
+ SearchLog.flag_likely_crawlers!(
+ window_start: BrowserPageviewEvent.retention_cutoff,
+ window_end: Time.zone.now,
+ )
+ puts "#{db}: flagged #{likely} more from sessions scored as likely crawlers"
+ end
+end
diff --git a/spec/jobs/detect_crawler_pageviews_spec.rb b/spec/jobs/detect_crawler_pageviews_spec.rb
index 40abf4e3401..a351c865c13 100644
--- a/spec/jobs/detect_crawler_pageviews_spec.rb
+++ b/spec/jobs/detect_crawler_pageviews_spec.rb
@@ -16,4 +16,20 @@ RSpec.describe Jobs::DetectCrawlerPageviews do
described_class.new.execute({})
end
+
+ it "flags searches made from sessions the scoring pass marked as likely crawlers" do
+ SiteSetting.improved_crawler_detection = true
+ session_id = SecureRandom.hex(16)
+ Fabricate(
+ :browser_pageview_event,
+ session_id: session_id,
+ user_agent: "Mozilla/5.0 (X11; Linux x86_64) HeadlessChrome/120.0.0.0",
+ created_at: 45.minutes.ago,
+ )
+ log = Fabricate(:search_log, user: nil, session_id: session_id, created_at: 45.minutes.ago)
+
+ described_class.new.execute({})
+
+ expect(log.reload.likely_crawler).to eq(true)
+ end
end
diff --git a/spec/lib/dashboard_rollup_rebuilder_spec.rb b/spec/lib/dashboard_rollup_rebuilder_spec.rb
new file mode 100644
index 00000000000..9738dda5f23
--- /dev/null
+++ b/spec/lib/dashboard_rollup_rebuilder_spec.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+RSpec.describe DashboardRollupRebuilder do
+ subject(:rebuilder) { described_class.new(io: StringIO.new) }
+
+ describe "#rebuild!" do
+ it "rebuilds country and referrer rollups from the events they summarise" do
+ Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")
+ BrowserPageviewCountryDailyRollup.aggregate(start_date: 1.day.ago, end_date: Time.zone.today)
+ BrowserPageviewReferrerDailyRollup.aggregate(start_date: 1.day.ago, end_date: Time.zone.today)
+ BrowserPageviewCountryDailyRollup.delete_all
+ BrowserPageviewReferrerDailyRollup.delete_all
+
+ rebuilder.rebuild!
+
+ expect(BrowserPageviewCountryDailyRollup.sum(:count)).to eq(1)
+ expect(BrowserPageviewReferrerDailyRollup.sum(:count)).to eq(1)
+ end
+
+ it "populates likely crawler counts on rollups that were built without them" do
+ SiteSetting.improved_crawler_detection = true
+ Fabricate(:browser_pageview_event, country_code: "US", score: 90)
+ BrowserPageviewCountryDailyRollup.aggregate(start_date: 1.day.ago, end_date: Time.zone.today)
+ BrowserPageviewCountryDailyRollup.update_all(likely_crawler_count: 0)
+
+ rebuilder.rebuild!("browser_pageview_country")
+
+ expect(BrowserPageviewCountryDailyRollup.sum(:likely_crawler_count)).to eq(1)
+ end
+
+ it "rebuilds only the named rollup" do
+ Fabricate(:browser_pageview_event, country_code: "US")
+
+ rebuilder.rebuild!("browser_pageview_country")
+
+ expect(BrowserPageviewCountryDailyRollup.count).to eq(1)
+ expect(BrowserPageviewReferrerDailyRollup.count).to eq(0)
+ end
+
+ it "leaves browser pageview rollups empty when no events exist" do
+ expect { rebuilder.rebuild!("browser_pageview_country") }.not_to change {
+ BrowserPageviewCountryDailyRollup.count
+ }
+ end
+
+ it "leaves user visit rollups empty when nobody has visited" do
+ UserVisit.delete_all
+
+ expect { rebuilder.rebuild!("user_visit") }.not_to change { UserVisitDailyRollup.count }
+ end
+ end
+
+ describe ".known?" do
+ it "recognises every rollup it can rebuild and rejects anything else" do
+ expect(described_class.names).to all(satisfy { |name| described_class.known?(name) })
+ expect(described_class.known?("nope")).to eq(false)
+ end
+ end
+end
diff --git a/spec/models/browser_pageview_country_daily_rollup_spec.rb b/spec/models/browser_pageview_country_daily_rollup_spec.rb
index 0564a6dde1c..65196c35e14 100644
--- a/spec/models/browser_pageview_country_daily_rollup_spec.rb
+++ b/spec/models/browser_pageview_country_daily_rollup_spec.rb
@@ -22,17 +22,21 @@ RSpec.describe BrowserPageviewCountryDailyRollup do
)
end
- it "splits total count from logged-in count" do
+ it "splits the total count by logged-in and likely crawler traffic" do
user = Fabricate(:user)
- Fabricate(:browser_pageview_event, country_code: "US", user_id: user.id)
- Fabricate(:browser_pageview_event, country_code: "US")
- Fabricate(:browser_pageview_event, country_code: "US")
+ Fabricate(:browser_pageview_event, country_code: "US", score: 90)
+ Fabricate(:browser_pageview_event, country_code: "US", score: 90, user_id: user.id)
+ Fabricate(:browser_pageview_event, country_code: "US", score: 10)
+ Fabricate(:browser_pageview_event, country_code: "US", score: nil, user_id: user.id)
described_class.aggregate(start_date: start_date, end_date: end_date)
- row = described_class.first
- expect(row.count).to eq(3)
- expect(row.logged_in_count).to eq(1)
+ expect(described_class.find_by(country_code: "US")).to have_attributes(
+ count: 4,
+ logged_in_count: 2,
+ likely_crawler_count: 2,
+ likely_crawler_logged_in_count: 1,
+ )
end
it "stores rows for NULL country_code (no GeoIP match) without duplication on re-run" do
diff --git a/spec/models/browser_pageview_referrer_daily_rollup_spec.rb b/spec/models/browser_pageview_referrer_daily_rollup_spec.rb
index c340cc65da2..3ba40b3a51a 100644
--- a/spec/models/browser_pageview_referrer_daily_rollup_spec.rb
+++ b/spec/models/browser_pageview_referrer_daily_rollup_spec.rb
@@ -16,16 +16,25 @@ RSpec.describe BrowserPageviewReferrerDailyRollup do
expect(rollups).to contain_exactly(["google.com", 2], ["reddit.com", 1])
end
- it "splits total count from logged-in count" do
+ it "splits the total count by logged-in and likely crawler traffic" do
user = Fabricate(:user)
- Fabricate(:browser_pageview_event, normalized_referrer: "google.com", user_id: user.id)
- Fabricate(:browser_pageview_event, normalized_referrer: "google.com")
+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com", score: 90)
+ Fabricate(
+ :browser_pageview_event,
+ normalized_referrer: "google.com",
+ score: 90,
+ user_id: user.id,
+ )
+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com", score: 10)
described_class.aggregate(start_date: start_date, end_date: end_date)
- row = described_class.find_by(normalized_referrer: "google.com")
- expect(row.count).to eq(2)
- expect(row.logged_in_count).to eq(1)
+ expect(described_class.find_by(normalized_referrer: "google.com")).to have_attributes(
+ count: 3,
+ logged_in_count: 1,
+ likely_crawler_count: 2,
+ likely_crawler_logged_in_count: 1,
+ )
end
it "stores NULL-referrer rows (direct visits)" do
diff --git a/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb b/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb
index 60dce237463..3b7572c26e5 100644
--- a/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb
+++ b/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb
@@ -285,5 +285,68 @@ RSpec.describe BrowserPageviewSessionEngagementDailyRollup do
have_attributes(sessions: 2, bounced: 2, engaged_seconds_total: 10),
)
end
+
+ it "records the likely crawler share of sessions, bounces, and engaged seconds" do
+ crawler_event =
+ Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 10, 9), score: 90)
+ Fabricate(
+ :browser_pageview_session_engagement,
+ session_id: crawler_event.session_id,
+ engaged_seconds: 3,
+ )
+
+ human_event =
+ Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 11, 10), score: 10)
+ Fabricate(
+ :browser_pageview_session_engagement,
+ session_id: human_event.session_id,
+ engaged_seconds: 40,
+ )
+
+ described_class.aggregate(start_date:, end_date:)
+
+ expect(described_class.all).to contain_exactly(
+ have_attributes(
+ date: Date.new(2026, 6, 10),
+ sessions: 1,
+ bounced: 1,
+ engaged_seconds_total: 3,
+ likely_crawler_sessions: 1,
+ likely_crawler_bounced: 1,
+ likely_crawler_engaged_seconds_total: 3,
+ ),
+ have_attributes(
+ date: Date.new(2026, 6, 11),
+ sessions: 1,
+ bounced: 0,
+ engaged_seconds_total: 40,
+ likely_crawler_sessions: 0,
+ likely_crawler_bounced: 0,
+ likely_crawler_engaged_seconds_total: 0,
+ ),
+ )
+ end
+
+ it "treats a session as a likely crawler when any of its pageviews scores above the threshold" do
+ session_id = SecureRandom.hex(16)
+ Fabricate(
+ :browser_pageview_event,
+ session_id: session_id,
+ created_at: Time.utc(2026, 6, 10, 9),
+ score: 10,
+ )
+ Fabricate(
+ :browser_pageview_event,
+ session_id: session_id,
+ created_at: Time.utc(2026, 6, 10, 10),
+ score: 90,
+ )
+
+ described_class.aggregate(start_date:, end_date:)
+
+ expect(described_class.all).to contain_exactly(
+ have_attributes(sessions: 1, likely_crawler_sessions: 1),
+ )
+ end
end
end
diff --git a/spec/models/category_activity_daily_rollup_spec.rb b/spec/models/category_activity_daily_rollup_spec.rb
index 555aaeea144..b6e0beab1b7 100644
--- a/spec/models/category_activity_daily_rollup_spec.rb
+++ b/spec/models/category_activity_daily_rollup_spec.rb
@@ -25,6 +25,111 @@ RSpec.describe CategoryActivityDailyRollup do
)
end
+ it "records likely crawler pageviews from browser events separately" do
+ topic = Fabricate(:topic, category: category, created_at: 3.days.ago)
+ TopicViewStat.create!(
+ topic: topic,
+ viewed_at: 3.days.ago.to_date,
+ anonymous_views: 7,
+ logged_in_views: 3,
+ )
+ Fabricate(
+ :browser_pageview_event,
+ topic_id: topic.id,
+ ip_address: "1.1.1.1",
+ created_at: 3.days.ago,
+ score: 90,
+ )
+ Fabricate(
+ :browser_pageview_event,
+ topic_id: topic.id,
+ ip_address: "2.2.2.2",
+ created_at: 3.days.ago,
+ score: 90,
+ )
+ Fabricate(:browser_pageview_event, topic_id: topic.id, created_at: 3.days.ago, score: 10)
+
+ described_class.aggregate(start_date: 5.days.ago, end_date: Time.zone.today)
+
+ expect(described_class.find_by(category: category, date: 3.days.ago.to_date)).to(
+ have_attributes(page_views: 10, likely_crawler_page_views: 2),
+ )
+ end
+
+ it "counts a likely crawler revisiting the same topic once a day, matching topic view stats" do
+ topic = Fabricate(:topic, category: category, created_at: 3.days.ago)
+ TopicViewStat.create!(
+ topic: topic,
+ viewed_at: 3.days.ago.to_date,
+ anonymous_views: 5,
+ logged_in_views: 0,
+ )
+ 2.times do
+ Fabricate(
+ :browser_pageview_event,
+ topic_id: topic.id,
+ ip_address: "1.1.1.1",
+ created_at: 3.days.ago,
+ score: 90,
+ )
+ end
+
+ described_class.aggregate(start_date: 5.days.ago, end_date: Time.zone.today)
+
+ expect(described_class.find_by(category: category, date: 3.days.ago.to_date)).to(
+ have_attributes(page_views: 5, likely_crawler_page_views: 1),
+ )
+ end
+
+ it "does not create a row for a day whose only signal is likely crawler pageviews" do
+ topic = Fabricate(:topic, category: category, created_at: 10.days.ago)
+ Fabricate(:browser_pageview_event, topic_id: topic.id, created_at: 3.days.ago, score: 90)
+
+ described_class.aggregate(start_date: 5.days.ago, end_date: Time.zone.today)
+
+ expect(described_class.where(date: 3.days.ago.to_date)).to be_empty
+ end
+
+ it "keeps likely crawler pageviews for dates whose source events have been pruned" do
+ topic = Fabricate(:topic, category: category, created_at: 3.days.ago)
+ TopicViewStat.create!(
+ topic: topic,
+ viewed_at: 3.days.ago.to_date,
+ anonymous_views: 10,
+ logged_in_views: 0,
+ )
+ Fabricate(:browser_pageview_event, topic_id: topic.id, created_at: 3.days.ago, score: 90)
+ described_class.aggregate(start_date: 5.days.ago, end_date: Time.zone.today)
+
+ BrowserPageviewEvent.delete_all
+ described_class.aggregate(start_date: 5.days.ago, end_date: Time.zone.today)
+
+ expect(described_class.find_by(category: category, date: 3.days.ago.to_date)).to(
+ have_attributes(page_views: 10, likely_crawler_page_views: 1),
+ )
+ end
+
+ it "recomputes likely crawler pageviews downward while the source events remain" do
+ topic = Fabricate(:topic, category: category, created_at: 3.days.ago)
+ TopicViewStat.create!(
+ topic: topic,
+ viewed_at: 3.days.ago.to_date,
+ anonymous_views: 10,
+ logged_in_views: 0,
+ )
+ crawler_event =
+ Fabricate(:browser_pageview_event, topic_id: topic.id, created_at: 3.days.ago, score: 90)
+ Fabricate(:browser_pageview_event, topic_id: topic.id, created_at: 3.days.ago, score: 10)
+ described_class.aggregate(start_date: 5.days.ago, end_date: Time.zone.today)
+
+ crawler_event.update!(score: 10)
+ described_class.aggregate(start_date: 5.days.ago, end_date: Time.zone.today)
+
+ expect(described_class.find_by(category: category, date: 3.days.ago.to_date)).to(
+ have_attributes(page_views: 10, likely_crawler_page_views: 0),
+ )
+ end
+
it "ignores deleted topics, deleted posts and private messages" do
Fabricate(:topic, category: category, created_at: 1.day.ago, deleted_at: Time.zone.now)
Fabricate(:private_message_topic, category: category, created_at: 1.day.ago)
@@ -139,6 +244,72 @@ RSpec.describe CategoryActivityDailyRollup do
expect(row.topics_current).to be_a(Integer)
end
+ it "subtracts likely crawler pageviews once crawler detection is enabled" do
+ SiteSetting.improved_crawler_detection = true
+ Fabricate(
+ :category_activity_daily_rollup,
+ category: category,
+ date: 2.days.ago,
+ topics: 0,
+ posts: 0,
+ page_views: 10,
+ likely_crawler_page_views: 4,
+ )
+
+ row =
+ described_class.period_totals(
+ prev_start: 14.days.ago.to_date,
+ current_start: 7.days.ago.to_date,
+ current_end: Time.zone.today,
+ ).first
+
+ expect(row.page_views_current).to eq(6)
+ end
+
+ it "clamps to zero when likely crawler pageviews overshoot the topic view stats" do
+ SiteSetting.improved_crawler_detection = true
+ Fabricate(
+ :category_activity_daily_rollup,
+ category: category,
+ date: 2.days.ago,
+ topics: 1,
+ posts: 0,
+ page_views: 3,
+ likely_crawler_page_views: 9,
+ )
+
+ row =
+ described_class.period_totals(
+ prev_start: 14.days.ago.to_date,
+ current_start: 7.days.ago.to_date,
+ current_end: Time.zone.today,
+ ).first
+
+ expect(row.page_views_current).to eq(0)
+ end
+
+ it "counts likely crawler pageviews while crawler detection is disabled" do
+ SiteSetting.improved_crawler_detection = false
+ Fabricate(
+ :category_activity_daily_rollup,
+ category: category,
+ date: 2.days.ago,
+ topics: 0,
+ posts: 0,
+ page_views: 10,
+ likely_crawler_page_views: 4,
+ )
+
+ row =
+ described_class.period_totals(
+ prev_start: 14.days.ago.to_date,
+ current_start: 7.days.ago.to_date,
+ current_end: Time.zone.today,
+ ).first
+
+ expect(row.page_views_current).to eq(10)
+ end
+
it "excludes categories outside the requested ids" do
other_category = Fabricate(:category)
Fabricate(:category_activity_daily_rollup, category: category)
diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rb
index c2fbad34dc8..4483b95b040 100644
--- a/spec/models/report_spec.rb
+++ b/spec/models/report_spec.rb
@@ -1974,6 +1974,19 @@ RSpec.describe Report do
end
end
+ describe ".cache_key" do
+ it "includes the crawler detection state in the cache key" do
+ report = Report.find("signups")
+
+ SiteSetting.improved_crawler_detection = false
+ disabled_key = Report.cache_key(report)
+
+ SiteSetting.improved_crawler_detection = true
+
+ expect(Report.cache_key(report)).not_to eq(disabled_key)
+ end
+ end
+
describe "top_uploads" do
context "with no data" do
it "works" do
diff --git a/spec/models/reports/top_countries_by_browser_pageviews_spec.rb b/spec/models/reports/top_countries_by_browser_pageviews_spec.rb
index 61131dc1f31..f345652d5aa 100644
--- a/spec/models/reports/top_countries_by_browser_pageviews_spec.rb
+++ b/spec/models/reports/top_countries_by_browser_pageviews_spec.rb
@@ -13,7 +13,7 @@ describe Reports::TopCountriesByBrowserPageviews do
it "ranks countries by event count and computes percent of total browser pageviews" do
3.times { Fabricate(:browser_pageview_event, country_code: "US") }
- 1.times { Fabricate(:browser_pageview_event, country_code: "GB") }
+ Fabricate(:browser_pageview_event, country_code: "GB")
data = report.data
expect(data.map { |row| row[:country_code] }).to eq(%w[US GB])
@@ -52,6 +52,25 @@ describe Reports::TopCountriesByBrowserPageviews do
expect(data.first[:percent]).to eq(50)
end
+ it "excludes likely crawler pageviews once crawler detection is enabled" do
+ SiteSetting.improved_crawler_detection = true
+ 3.times { Fabricate(:browser_pageview_event, country_code: "US", score: 10) }
+ Fabricate(:browser_pageview_event, country_code: "GB", score: 90)
+ Fabricate(:browser_pageview_event, country_code: "GB", score: 10)
+
+ data = report.data
+ expect(data.map { |row| row[:country_code] }).to eq(%w[US GB])
+ expect(data.map { |row| row[:count] }).to eq([3, 1])
+ expect(data.first[:percent]).to eq(75)
+ end
+
+ it "counts likely crawler pageviews while crawler detection is disabled" do
+ SiteSetting.improved_crawler_detection = false
+ 3.times { Fabricate(:browser_pageview_event, country_code: "US", score: 90) }
+
+ expect(report.data.first[:count]).to eq(3)
+ end
+
it "returns empty data when no events exist in range" do
expect(report.data).to eq([])
end
diff --git a/spec/models/reports/top_referrers_by_browser_pageviews_spec.rb b/spec/models/reports/top_referrers_by_browser_pageviews_spec.rb
index 1e19d459e13..42d2bd4693a 100644
--- a/spec/models/reports/top_referrers_by_browser_pageviews_spec.rb
+++ b/spec/models/reports/top_referrers_by_browser_pageviews_spec.rb
@@ -17,7 +17,7 @@ describe Reports::TopReferrersByBrowserPageviews do
3.times do
Fabricate(:browser_pageview_event, normalized_referrer: "news.ycombinator.com/item?id=1")
end
- 1.times { Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com/r/discourse") }
+ Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com/r/discourse")
data = report.data
expect(data.map { |row| row[:normalized_referrer] }).to eq(
@@ -91,6 +91,25 @@ describe Reports::TopReferrersByBrowserPageviews do
expect(report.data.first[:count]).to eq(1)
end
+ it "excludes likely crawler pageviews once crawler detection is enabled" do
+ SiteSetting.improved_crawler_detection = true
+ 3.times { Fabricate(:browser_pageview_event, normalized_referrer: "google.com", score: 10) }
+ Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com", score: 90)
+ Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com", score: 10)
+
+ data = report.data
+ expect(data.map { |row| row[:normalized_referrer] }).to eq(%w[google.com reddit.com])
+ expect(data.map { |row| row[:count] }).to eq([3, 1])
+ expect(data.first[:percent]).to eq(75)
+ end
+
+ it "counts likely crawler pageviews while crawler detection is disabled" do
+ SiteSetting.improved_crawler_detection = false
+ 3.times { Fabricate(:browser_pageview_event, normalized_referrer: "google.com", score: 90) }
+
+ expect(report.data.first[:count]).to eq(3)
+ end
+
it "returns empty data when no qualifying events exist" do
Fabricate(:browser_pageview_event, normalized_referrer: nil)
expect(report.data).to eq([])
diff --git a/spec/models/search_log_spec.rb b/spec/models/search_log_spec.rb
index 7f147e75041..35d965333af 100644
--- a/spec/models/search_log_spec.rb
+++ b/spec/models/search_log_spec.rb
@@ -187,6 +187,182 @@ RSpec.describe SearchLog, type: :model do
end
end
+ describe ".log" do
+ it "flags an anonymous search from a known crawler user agent" do
+ _status, id =
+ SearchLog.log(
+ term: "ruby",
+ search_type: :header,
+ ip_address: "127.0.0.1",
+ user_agent: "Googlebot/2.1 (+http://www.google.com/bot.html)",
+ )
+
+ expect(SearchLog.find(id).crawler).to eq(true)
+ end
+
+ it "flags a logged-in search from a crawler user agent" do
+ user = Fabricate(:user)
+ _status, id =
+ SearchLog.log(
+ term: "ruby",
+ search_type: :header,
+ ip_address: "127.0.0.1",
+ user_agent: "Googlebot/2.1 (+http://www.google.com/bot.html)",
+ user_id: user.id,
+ )
+
+ expect(SearchLog.find(id).crawler).to eq(true)
+ end
+
+ it "flags a search from a scripted client that is not a browser" do
+ _status, id =
+ SearchLog.log(
+ term: "ruby",
+ search_type: :header,
+ ip_address: "127.0.0.1",
+ user_agent: "python-requests/2.31.0",
+ )
+
+ expect(SearchLog.find(id).crawler).to eq(true)
+ end
+
+ it "records the pageview session the search was made from" do
+ _status, id =
+ SearchLog.log(
+ term: "ruby",
+ search_type: :header,
+ ip_address: "127.0.0.1",
+ session_id: "a" * (SearchLog::MAXIMUM_SESSION_ID_LENGTH + 10),
+ )
+
+ expect(SearchLog.find(id).session_id).to eq("a" * SearchLog::MAXIMUM_SESSION_ID_LENGTH)
+ end
+
+ it "does not flag an anonymous search that arrived without a user agent" do
+ _status, id = SearchLog.log(term: "ruby", search_type: :header, ip_address: "127.0.0.1")
+
+ expect(SearchLog.find(id).crawler).to eq(false)
+ end
+
+ it "does not flag an anonymous search from a browser user agent" do
+ _status, id =
+ SearchLog.log(
+ term: "ruby",
+ search_type: :header,
+ ip_address: "127.0.0.1",
+ user_agent:
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
+ )
+
+ expect(SearchLog.find(id).crawler).to eq(false)
+ end
+ end
+
+ describe ".flag_likely_crawlers!" do
+ let(:session_id) { SecureRandom.hex(16) }
+ let(:crawler_score) { CrawlerScorer::BOT_SCORE_THRESHOLD + 1 }
+
+ def flag!
+ described_class.flag_likely_crawlers!(window_start: 1.hour.ago, window_end: Time.now)
+ end
+
+ it "flags a search made from a session scored as a likely crawler" do
+ Fabricate(:browser_pageview_event, session_id: session_id, score: crawler_score)
+ log = Fabricate(:search_log, user: nil, session_id: session_id, created_at: 30.minutes.ago)
+
+ flag!
+
+ expect(log.reload.likely_crawler).to eq(true)
+ end
+
+ it "leaves a search whose session was scored below the crawler threshold" do
+ Fabricate(
+ :browser_pageview_event,
+ session_id: session_id,
+ score: CrawlerScorer::BOT_SCORE_THRESHOLD,
+ )
+ log = Fabricate(:search_log, user: nil, session_id: session_id, created_at: 30.minutes.ago)
+
+ flag!
+
+ expect(log.reload.likely_crawler).to eq(false)
+ end
+
+ it "leaves a search that carries no session" do
+ Fabricate(:browser_pageview_event, session_id: session_id, score: crawler_score)
+ log = Fabricate(:search_log, user: nil, session_id: nil, created_at: 30.minutes.ago)
+
+ flag!
+
+ expect(log.reload.likely_crawler).to eq(false)
+ end
+
+ it "leaves a search whose session has no scored pageviews at all" do
+ Fabricate(:browser_pageview_event, session_id: SecureRandom.hex(16), score: crawler_score)
+ log = Fabricate(:search_log, user: nil, session_id: session_id, created_at: 30.minutes.ago)
+
+ flag!
+
+ expect(log.reload.likely_crawler).to eq(false)
+ end
+
+ it "flags a logged-in search whose session is scored as a likely crawler" do
+ Fabricate(:browser_pageview_event, session_id: session_id, score: crawler_score)
+ log =
+ Fabricate(
+ :search_log,
+ user: Fabricate(:user),
+ ip_address: nil,
+ session_id: session_id,
+ created_at: 30.minutes.ago,
+ )
+
+ flag!
+
+ expect(log.reload.likely_crawler).to eq(true)
+ end
+
+ it "leaves searches outside the window alone" do
+ Fabricate(:browser_pageview_event, session_id: session_id, score: crawler_score)
+ log = Fabricate(:search_log, user: nil, session_id: session_id, created_at: 5.hours.ago)
+
+ flag!
+
+ expect(log.reload.likely_crawler).to eq(false)
+ end
+ end
+
+ describe ".backfill_crawler!" do
+ it "flags existing searches from known crawler user agents" do
+ crawler =
+ Fabricate(
+ :search_log,
+ user: nil,
+ user_agent: "Googlebot/2.1 (+http://www.google.com/bot.html)",
+ )
+ human = Fabricate(:search_log, user: nil, user_agent: "Mozilla/5.0 (Macintosh) Chrome/120.0")
+ no_agent = Fabricate(:search_log, user: nil, user_agent: nil)
+ blank_agent = Fabricate(:search_log, user: nil, user_agent: "")
+ member = Fabricate(:search_log, user: Fabricate(:user), user_agent: "Googlebot/2.1")
+
+ expect(described_class.backfill_crawler!).to eq(2)
+
+ expect(crawler.reload.crawler).to eq(true)
+ expect(human.reload.crawler).to eq(false)
+ expect(no_agent.reload.crawler).to eq(false)
+ expect(blank_agent.reload.crawler).to eq(false)
+ expect(member.reload.crawler).to eq(true)
+ end
+
+ it "is idempotent" do
+ Fabricate(:search_log, user: nil, user_agent: "Googlebot/2.1")
+
+ described_class.backfill_crawler!
+
+ expect(described_class.backfill_crawler!).to eq(0)
+ end
+ end
+
describe ".term_details" do
it "should only use the date for the period" do
time = Time.utc(2019, 5, 23, 18, 15, 30)
@@ -241,17 +417,42 @@ RSpec.describe SearchLog, type: :model do
SearchLog.term_details("ruby", :weekly, :non_staff_only)[:data].sum { |point| point[:y] },
).to eq(1)
end
+
+ it "returns non-staff and anonymous searches minus crawlers with the human_only search type" do
+ SiteSetting.improved_crawler_detection = true
+ member = Fabricate(:user)
+ Fabricate(:search_log, term: "ruby", user: member)
+ Fabricate(:search_log, term: "ruby", user: Fabricate(:admin))
+ Fabricate(:search_log, term: "ruby", user: Fabricate(:moderator))
+ Fabricate(:search_log, term: "ruby", user: nil)
+ Fabricate(:search_log, term: "ruby", user: nil, crawler: true)
+
+ expect(
+ SearchLog.term_details("ruby", :weekly, :human_only)[:data].sum { |point| point[:y] },
+ ).to eq(2)
+ end
end
describe "trending" do
fab!(:user)
+ def log_search(term, **opts)
+ SearchLog.log(
+ term: term,
+ search_type: :header,
+ ip_address: "127.0.0.1",
+ user_agent:
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
+ **opts,
+ )
+ end
+
before do
- SearchLog.log(term: "ruby", search_type: :header, ip_address: "127.0.0.1")
- SearchLog.log(term: "php", search_type: :header, ip_address: "127.0.0.1")
- SearchLog.log(term: "java", search_type: :header, ip_address: "127.0.0.1")
- SearchLog.log(term: "ruby", search_type: :header, ip_address: "127.0.0.1", user_id: user.id)
- SearchLog.log(term: "swift", search_type: :header, ip_address: "127.0.0.1")
- SearchLog.log(term: "ruby", search_type: :header, ip_address: "127.0.0.2")
+ log_search("ruby")
+ log_search("php")
+ log_search("java")
+ log_search("ruby", user_id: user.id)
+ log_search("swift")
+ log_search("ruby", ip_address: "127.0.0.2")
end
it "considers time period" do
@@ -285,6 +486,28 @@ RSpec.describe SearchLog, type: :model do
expect(results.map { |trend| [trend.term, trend.searches] }).to eq([["ruby", 1]])
end
+
+ it "returns non-staff and anonymous searches minus crawlers with the human_only search type" do
+ SiteSetting.improved_crawler_detection = true
+ Fabricate(:search_log, term: "admin-search", user: Fabricate(:admin))
+ Fabricate(:search_log, term: "crawler-search", user: nil, crawler: true)
+
+ results = SearchLog.trending(:all, :human_only).to_a
+
+ expect(results.map { |trend| [trend.term, trend.searches] }).to eq(
+ [["ruby", 3], ["java", 1], ["php", 1], ["swift", 1]],
+ )
+ end
+
+ it "falls back to members-only for human_only while crawler detection is disabled" do
+ SiteSetting.improved_crawler_detection = false
+ Fabricate(:search_log, term: "admin-search", user: Fabricate(:admin))
+ Fabricate(:search_log, term: "anonymous-search", user: nil)
+
+ results = SearchLog.trending(:all, :human_only).to_a
+
+ expect(results.map { |trend| [trend.term, trend.searches] }).to eq([["ruby", 1]])
+ end
end
describe "clean_up" do
diff --git a/spec/requests/admin/dashboard_controller_spec.rb b/spec/requests/admin/dashboard_controller_spec.rb
index 423d8f20be3..7a3054d07ff 100644
--- a/spec/requests/admin/dashboard_controller_spec.rb
+++ b/spec/requests/admin/dashboard_controller_spec.rb
@@ -336,6 +336,7 @@ RSpec.describe Admin::DashboardController do
expect(response.status).to eq(200)
expect(search_data).to eq(
"logging_enabled" => true,
+ "search_type" => "non_staff_only",
"kpis" => {
"total_searches" => {
"value" => 2,
diff --git a/spec/requests/admin/search_logs_spec.rb b/spec/requests/admin/search_logs_spec.rb
index 1946eaf5efe..581a4fc0aba 100644
--- a/spec/requests/admin/search_logs_spec.rb
+++ b/spec/requests/admin/search_logs_spec.rb
@@ -50,6 +50,20 @@ RSpec.describe Admin::SearchLogsController do
[["member-search", 1]],
)
end
+
+ it "returns non-staff and anonymous searches minus crawlers with the human_only search type" do
+ SiteSetting.improved_crawler_detection = true
+ Fabricate(:search_log, term: "member-search", user: user)
+ Fabricate(:search_log, term: "admin-search", user: admin)
+ Fabricate(:search_log, term: "anonymous-search", user: nil)
+ Fabricate(:search_log, term: "crawler-search", user: nil, crawler: true)
+
+ get "/admin/logs/search_logs.json", params: { search_type: "human_only" }
+
+ expect(response.parsed_body.map { |entry| [entry["term"], entry["searches"]] }).to eq(
+ [["anonymous-search", 1], ["member-search", 1], ["ruby", 1]],
+ )
+ end
end
context "when logged in as a moderator" do
diff --git a/spec/requests/search_controller_spec.rb b/spec/requests/search_controller_spec.rb
index bd2b4599e32..2a51527944b 100644
--- a/spec/requests/search_controller_spec.rb
+++ b/spec/requests/search_controller_spec.rb
@@ -275,6 +275,22 @@ RSpec.describe SearchController do
expect(log.term).to eq("wookie")
end
+ it "logs the pageview session the search was made from" do
+ SiteSetting.log_search_queries = true
+ session_id = SecureRandom.alphanumeric(32)
+
+ get "/search/query.json",
+ params: {
+ term: "wookie",
+ },
+ headers: {
+ "Discourse-Pageview-Session-Id" => session_id,
+ }
+
+ expect(response.status).to eq(200)
+ expect(SearchLog.find_by(term: "wookie").session_id).to eq(session_id)
+ end
+
it "doesn't log when disabled" do
SiteSetting.log_search_queries = false
get "/search/query.json", params: { term: "wookie" }
diff --git a/spec/services/admin_dashboard_engagement_spec.rb b/spec/services/admin_dashboard_engagement_spec.rb
index 54c520f6edc..db06c62c410 100644
--- a/spec/services/admin_dashboard_engagement_spec.rb
+++ b/spec/services/admin_dashboard_engagement_spec.rb
@@ -237,6 +237,12 @@ describe AdminDashboardEngagement do
end
describe "activity_by_category" do
+ def page_views_for(result, category)
+ result[:activity_by_category][:rows].find { |row| row[:category_id] == category.id }[
+ :page_views
+ ]
+ end
+
it "includes the activity_by_category block with rows and total" do
result = described_class.build(start_date: "2026-04-01", end_date: "2026-04-28")
activity = result[:activity_by_category]
@@ -245,6 +251,27 @@ describe AdminDashboardEngagement do
expect(activity).to have_key(:total)
end
+ it "invalidates the cached rows when crawler detection is toggled" do
+ SiteSetting.improved_crawler_detection = false
+ category = Fabricate(:category)
+ Fabricate(
+ :category_activity_daily_rollup,
+ category: category,
+ date: "2026-04-10",
+ topics: 0,
+ posts: 0,
+ page_views: 10,
+ likely_crawler_page_views: 4,
+ )
+
+ first = described_class.build(start_date: "2026-04-01", end_date: "2026-04-28")
+ expect(page_views_for(first, category)).to eq(10)
+
+ SiteSetting.improved_crawler_detection = true
+ second = described_class.build(start_date: "2026-04-01", end_date: "2026-04-28")
+ expect(page_views_for(second, category)).to eq(6)
+ end
+
it "honours category visibility when current_user is a moderator" do
moderator = Fabricate(:moderator)
private_group = Fabricate(:group)
diff --git a/spec/services/admin_dashboard_search_spec.rb b/spec/services/admin_dashboard_search_spec.rb
index bd41283cecf..cde7fdb9537 100644
--- a/spec/services/admin_dashboard_search_spec.rb
+++ b/spec/services/admin_dashboard_search_spec.rb
@@ -3,10 +3,13 @@
RSpec.describe AdminDashboardSearch do
fab!(:user)
- before { freeze_time(Time.zone.local(2026, 5, 14, 12, 0, 0)) }
+ before do
+ freeze_time(Time.zone.local(2026, 5, 14, 12, 0, 0))
+ SiteSetting.improved_crawler_detection = true
+ end
describe ".build" do
- it "returns KPIs, trending terms, and content gaps for the selected dates, counting only non-staff users" do
+ it "returns KPIs, trending terms, and content gaps for the selected dates, counting non-staff and anonymous searches" do
admin = Fabricate(:admin)
moderator = Fabricate(:moderator)
@@ -55,13 +58,26 @@ RSpec.describe AdminDashboardSearch do
)
Fabricate.times(5, :search_log, term: "ghost", user: user, created_at: "2026-04-26 11:00")
- # Anonymous searches (likely crawlers) must be excluded from every metric and from the
+ Fabricate.times(8, :clicked_search_log, term: "onboarding", created_at: "2026-05-05 10:00")
+ Fabricate.times(2, :search_log, term: "onboarding", created_at: "2026-05-05 11:00")
+
+ # Searches flagged as crawler traffic must stay out of every metric and out of the
# prior-window deltas. If counted, "crawler-bait" would top trending, inflate the
- # no-result rate, and surface as a content gap; the anonymous "ruby" rows would change
- # its count and the percent change.
- Fabricate.times(30, :search_log, term: "crawler-bait", created_at: "2026-05-05 10:00")
- Fabricate.times(10, :search_log, term: "ruby", created_at: "2026-05-02 09:00")
- Fabricate.times(20, :search_log, term: "crawler-bait", created_at: "2026-04-26 09:00")
+ # no-result rate, and surface as a content gap.
+ Fabricate.times(
+ 30,
+ :search_log,
+ term: "crawler-bait",
+ crawler: true,
+ created_at: "2026-05-05 12:00",
+ )
+ Fabricate.times(
+ 20,
+ :search_log,
+ term: "crawler-bait",
+ crawler: true,
+ created_at: "2026-04-26 09:00",
+ )
Fabricate(:search_log, term: "admin-search", user: admin, created_at: "2026-05-05 12:00")
Fabricate(:clicked_search_log, term: "ruby", user: moderator, created_at: "2026-05-05 13:00")
@@ -73,20 +89,22 @@ RSpec.describe AdminDashboardSearch do
)
expect(described_class.build(start_date: "2026-05-01", end_date: "2026-05-07")).to eq(
logging_enabled: true,
+ search_type: "human_only",
kpis: {
total_searches: {
- value: 19,
+ value: 29,
previous_value: 10,
- percent_change: 90,
+ percent_change: 190,
},
no_result_rate: {
- value: 21,
+ value: 14,
previous_value: 50,
- point_change: -29,
+ point_change: -36,
exceeds_threshold: true,
},
},
trending: [
+ { term: "onboarding", searches: 10 },
{ term: "ruby", searches: 6 },
{ term: "markdown tables", searches: 5 },
{ term: "zeta", searches: 4 },
@@ -100,6 +118,19 @@ RSpec.describe AdminDashboardSearch do
)
end
+ it "stays members-only while crawler detection is disabled" do
+ SiteSetting.improved_crawler_detection = false
+ Fabricate.times(3, :search_log, term: "ruby", user: user, created_at: "2026-05-02 10:00")
+ Fabricate.times(9, :search_log, term: "crawler-bait", created_at: "2026-05-02 11:00")
+ Fabricate.times(4, :search_log, term: "onboarding", created_at: "2026-05-02 12:00")
+
+ result = described_class.build(start_date: "2026-05-01", end_date: "2026-05-07")
+
+ expect(result[:kpis][:total_searches][:value]).to eq(3)
+ expect(result[:trending]).to eq([{ term: "ruby", searches: 3 }])
+ expect(result[:search_type]).to eq("non_staff_only")
+ end
+
it "buckets terms by exact CTR boundaries" do
Fabricate(:clicked_search_log, term: "tiny-ctr", user: user, created_at: "2026-05-02 10:00")
Fabricate.times(
@@ -156,6 +187,7 @@ RSpec.describe AdminDashboardSearch do
expect(described_class.build(start_date: "2026-05-01", end_date: "2026-05-07")).to eq(
logging_enabled: true,
+ search_type: "human_only",
kpis: {
total_searches: {
value: 136,
@@ -196,6 +228,7 @@ RSpec.describe AdminDashboardSearch do
expect(described_class.build(start_date: "2026-05-01", end_date: "2026-05-07")).to eq(
logging_enabled: true,
+ search_type: "human_only",
kpis: {
total_searches: {
value: 11,
@@ -243,6 +276,7 @@ RSpec.describe AdminDashboardSearch do
expect(described_class.build(start_date: "2026-05-01", end_date: "2026-05-07")).to eq(
logging_enabled: true,
+ search_type: "human_only",
kpis: {
total_searches: {
value: 0,
diff --git a/spec/services/admin_dashboard_site_traffic_spec.rb b/spec/services/admin_dashboard_site_traffic_spec.rb
index 90f5f2fe73c..dc5f1eecfb4 100644
--- a/spec/services/admin_dashboard_site_traffic_spec.rb
+++ b/spec/services/admin_dashboard_site_traffic_spec.rb
@@ -728,6 +728,20 @@ RSpec.describe AdminDashboardSiteTraffic do
expect(second[:top_countries][:rows]).to be_empty
end
+ it "invalidates the cached payload when crawler detection is toggled" do
+ SiteSetting.improved_crawler_detection = false
+ 3.times { Fabricate(:browser_pageview_event, country_code: "US", score: 90) }
+ Fabricate(:browser_pageview_event, country_code: "US", score: 10)
+ aggregate_rollups
+
+ first = build_traffic(start_date: nil, end_date: nil)
+ expect(first[:top_countries][:rows].first[:count]).to eq(4)
+
+ SiteSetting.improved_crawler_detection = true
+ second = build_traffic(start_date: nil, end_date: nil)
+ expect(second[:top_countries][:rows].first[:count]).to eq(1)
+ end
+
it "invalidates the cached payload when current_hostname changes" do
Discourse.stubs(:current_hostname).returns("forum-a.example.com")
Fabricate(:browser_pageview_event, normalized_referrer: "forum-b.example.com/path")
@@ -827,6 +841,39 @@ RSpec.describe AdminDashboardSiteTraffic do
)
end
+ it "excludes likely crawler pageviews from the direct traffic share once crawler detection is enabled" do
+ SiteSetting.improved_crawler_detection = true
+ 3.times do
+ Fabricate(
+ :browser_pageview_event,
+ normalized_referrer: nil,
+ score: 10,
+ created_at: "2026-05-01",
+ )
+ end
+ 9.times do
+ Fabricate(
+ :browser_pageview_event,
+ normalized_referrer: "google.com",
+ score: 90,
+ created_at: "2026-05-01",
+ )
+ end
+ Fabricate(
+ :browser_pageview_event,
+ normalized_referrer: "google.com",
+ score: 10,
+ created_at: "2026-05-01",
+ )
+ aggregate_referrer_rollups
+
+ expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")[:kpis]).to include(
+ direct_traffic: {
+ value: 75,
+ },
+ )
+ end
+
it "reports zero direct traffic when every tracked pageview had a referrer" do
4.times do
Fabricate(
@@ -987,6 +1034,54 @@ RSpec.describe AdminDashboardSiteTraffic do
)
end
+ it "excludes likely crawler sessions once crawler detection is enabled" do
+ SiteSetting.improved_crawler_detection = true
+ Fabricate(
+ :browser_pageview_session_engagement_daily_rollup,
+ date: Date.new(2026, 5, 10),
+ logged_in: false,
+ sessions: 14,
+ bounced: 11,
+ engaged_seconds_total: 700,
+ likely_crawler_sessions: 6,
+ likely_crawler_bounced: 6,
+ likely_crawler_engaged_seconds_total: 100,
+ )
+
+ expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-14")[:kpis]).to include(
+ bounce_rate: {
+ value: 63,
+ },
+ average_session_duration_seconds: {
+ value: 75,
+ },
+ )
+ end
+
+ it "counts likely crawler sessions while crawler detection is disabled" do
+ SiteSetting.improved_crawler_detection = false
+ Fabricate(
+ :browser_pageview_session_engagement_daily_rollup,
+ date: Date.new(2026, 5, 10),
+ logged_in: false,
+ sessions: 14,
+ bounced: 11,
+ engaged_seconds_total: 700,
+ likely_crawler_sessions: 6,
+ likely_crawler_bounced: 6,
+ likely_crawler_engaged_seconds_total: 100,
+ )
+
+ expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-14")[:kpis]).to include(
+ bounce_rate: {
+ value: 79,
+ },
+ average_session_duration_seconds: {
+ value: 50,
+ },
+ )
+ end
+
it "reports zero average duration, not a placeholder, for visits with no engaged time" do
Fabricate(
:browser_pageview_session_engagement_daily_rollup,
@@ -1175,7 +1270,7 @@ RSpec.describe AdminDashboardSiteTraffic do
)
end
- it "excludes anonymous crawlers when login is required" do
+ it "excludes anonymous likely crawlers when login is required" do
SiteSetting.login_required = true
Fabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 10)
diff --git a/spec/system/admin_dashboard_redesign/search_section_spec.rb b/spec/system/admin_dashboard_redesign/search_section_spec.rb
index f37b0b57970..902f09f39a6 100644
--- a/spec/system/admin_dashboard_redesign/search_section_spec.rb
+++ b/spec/system/admin_dashboard_redesign/search_section_spec.rb
@@ -9,6 +9,7 @@ describe "Admin Dashboard Redesign | Search section" do
before do
SiteSetting.dashboard_improvements = true
+ SiteSetting.improved_crawler_detection = true
AdminDashboardSectionConfiguration.update(
[
{ id: "search", visible: true },
@@ -22,51 +23,42 @@ describe "Admin Dashboard Redesign | Search section" do
sign_in(current_user)
end
- it "lets staff review non-staff search health, inspect tooltips, and drill into terms",
+ it "lets staff review search health, inspect tooltips, and drill into terms",
time: Time.zone.local(2026, 5, 14, 12, 0, 0) do
Fabricate.times(
- 15,
+ 2,
:clicked_search_log,
term: "ruby",
user: user,
created_at: "2026-05-10 10:00",
)
- Fabricate.times(10, :search_log, term: "ruby", user: user, created_at: "2026-05-10 11:00")
- Fabricate.times(5, :search_log, term: "ruby", user: user, created_at: "2026-04-20 10:00")
+ Fabricate.times(3, :search_log, term: "ruby", user: user, created_at: "2026-05-10 11:00")
+ Fabricate(:search_log, term: "ruby", user: user, created_at: "2026-04-20 10:00")
- Fabricate.times(
- 2,
+ Fabricate(
:clicked_search_log,
term: "markdown tables",
user: user,
created_at: "2026-04-20 11:00",
)
-
- Fabricate.times(
- 13,
- :search_log,
- term: "markdown tables",
- user: user,
- created_at: "2026-04-20 12:00",
- )
Fabricate.times(
3,
:search_log,
term: "markdown tables",
user: user,
- created_at: "2026-05-03 10:00",
+ created_at: "2026-04-20 12:00",
)
+ Fabricate(:search_log, term: "markdown tables", user: user, created_at: "2026-05-03 10:00")
- Fabricate.times(2, :search_log, term: "discobot", user: user, created_at: "2026-05-10 12:00")
+ Fabricate(:search_log, term: "discobot", user: user, created_at: "2026-04-20 13:00")
Fabricate.times(
- 20,
+ 6,
:clicked_search_log,
term: "ruby",
user: user,
created_at: "2026-03-20 10:00",
)
- Fabricate.times(20, :search_log, term: "ruby", user: user, created_at: "2026-03-20 11:00")
Fabricate(:search_log, term: "admin-search", user: current_user, created_at: "2026-05-10 08:00")
Fabricate(:clicked_search_log, term: "ruby", user: moderator, created_at: "2026-05-10 08:30")
@@ -77,10 +69,22 @@ describe "Admin Dashboard Redesign | Search section" do
created_at: "2026-03-20 08:00",
)
- # Anonymous searches (likely crawlers) must be excluded from every metric. If they
+ # Searches flagged as crawler traffic must be excluded from every metric. If they
# were counted, "crawlerbot" would top trending and the no-result rate would spike.
- Fabricate.times(100, :search_log, term: "crawlerbot", created_at: "2026-05-10 09:00")
- Fabricate.times(50, :clicked_search_log, term: "ruby", created_at: "2026-05-10 09:30")
+ Fabricate.times(
+ 10,
+ :search_log,
+ term: "crawlerbot",
+ crawler: true,
+ created_at: "2026-05-10 09:00",
+ )
+ Fabricate.times(
+ 2,
+ :clicked_search_log,
+ term: "ruby",
+ crawler: true,
+ created_at: "2026-05-10 09:30",
+ )
dashboard.visit
expect(dashboard).to have_section("search")
@@ -93,12 +97,12 @@ describe "Admin Dashboard Redesign | Search section" do
"increased. Review the content gaps to see what's missing.",
)
- expect(search).to have_total_searches_kpi("50", improving_delta: "+25%")
- expect(search).to have_no_result_rate_kpi("4%", worsening_delta: "+4%")
+ expect(search).to have_total_searches_kpi("12", improving_delta: "+100%")
+ expect(search).to have_no_result_rate_kpi("8%", worsening_delta: "+8%")
search.hover_total_searches_tooltip
expect(search).to have_total_searches_tooltip(
- "The number of searches performed by members in your community.",
+ "The number of searches performed in your community.",
)
search.hover_no_result_rate_tooltip
@@ -108,13 +112,13 @@ describe "Admin Dashboard Redesign | Search section" do
)
search.hover_trending_tooltip
- expect(search).to have_trending_tooltip("The most popular search terms among members.")
+ expect(search).to have_trending_tooltip("The most popular search terms in your community.")
expect(search).to have_trending_rows(
[
- { term: "ruby", searches: 30 },
- { term: "markdown tables", searches: 18 },
- { term: "discobot", searches: 2 },
+ { term: "ruby", searches: 6 },
+ { term: "markdown tables", searches: 5 },
+ { term: "discobot", searches: 1 },
],
)
expect(search).to have_no_trending_term("admin-search")
@@ -122,8 +126,8 @@ describe "Admin Dashboard Redesign | Search section" do
expect(search).to have_content_gap_rows(
[
- { term: "markdown tables", searches: 18, badge: "Poor match" },
- { term: "discobot", searches: 2, badge: "No match" },
+ { term: "markdown tables", searches: 5, badge: "Poor match" },
+ { term: "discobot", searches: 1, badge: "No match" },
],
)
@@ -145,14 +149,14 @@ describe "Admin Dashboard Redesign | Search section" do
"looking for more often.",
)
- expect(search).to have_total_searches_kpi("27", improving_delta: "+800%")
- expect(search).to have_no_result_rate_kpi("7%", improving_delta: "-93%")
+ expect(search).to have_total_searches_kpi("5", improving_delta: "+400%")
+ expect(search).to have_no_result_rate_kpi("0%", improving_delta: "-100%")
search.click_trending_term("ruby")
expect(page).to have_current_path("/admin/logs/search_logs/term", ignore_query: true)
expect(Rack::Utils.parse_query(URI.parse(page.current_url).query)).to eq(
- "searchType" => "non_staff_only",
+ "searchType" => "human_only",
"period" => "weekly",
"term" => "ruby",
)
@@ -181,6 +185,7 @@ describe "Admin Dashboard Redesign | Search section" do
user: user,
created_at: "2026-05-10 11:00",
)
+ Fabricate.times(2, :search_log, term: "ghost", created_at: "2026-05-10 12:00")
dashboard.visit
search = dashboard.search
@@ -190,8 +195,36 @@ describe "Admin Dashboard Redesign | Search section" do
"Members are conducting more searches in your community, but the no-result rate has " \
"increased. Review the content gaps to see what's missing.",
)
- expect(search).to have_total_searches_kpi("10")
- expect(search).to have_alert_no_result_rate_kpi("50%")
+ expect(search).to have_total_searches_kpi("12")
+ expect(search).to have_alert_no_result_rate_kpi("58%")
+ end
+
+ it "stays members-only and drills into member searches while crawler detection is disabled",
+ time: Time.zone.local(2026, 5, 14, 12, 0, 0) do
+ SiteSetting.improved_crawler_detection = false
+ Fabricate.times(
+ 4,
+ :clicked_search_log,
+ term: "ruby",
+ user: user,
+ created_at: "2026-05-10 10:00",
+ )
+ Fabricate.times(2, :search_log, term: "crawlerbait", created_at: "2026-05-10 11:00")
+
+ dashboard.visit
+ search = dashboard.search
+
+ expect(search).to have_total_searches_kpi("4")
+ expect(search).to have_trending_rows([{ term: "ruby", searches: 4 }])
+ expect(search).to have_no_trending_term("crawlerbait")
+
+ search.click_trending_term("ruby")
+
+ expect(Rack::Utils.parse_query(URI.parse(page.current_url).query)).to eq(
+ "searchType" => "non_staff_only",
+ "period" => "monthly",
+ "term" => "ruby",
+ )
end
it "shows staff a graceful empty state when no searches were logged" do
@@ -257,7 +290,7 @@ describe "Admin Dashboard Redesign | Search section" do
expect(page).to have_current_path("/admin/logs/search_logs/term", ignore_query: true)
expect(Rack::Utils.parse_query(URI.parse(page.current_url).query)).to eq(
- "searchType" => "non_staff_only",
+ "searchType" => "human_only",
"period" => "all",
"term" => "ruby",
)
diff --git a/spec/tasks/admin_dashboard_spec.rb b/spec/tasks/admin_dashboard_spec.rb
new file mode 100644
index 00000000000..792eb3e09bd
--- /dev/null
+++ b/spec/tasks/admin_dashboard_spec.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+describe "admin_dashboard:rebuild_rollups" do
+ it "rebuilds every rollup when no name is given" do
+ DashboardRollupRebuilder.expects(:rebuild!).with(nil)
+
+ capture_stdout { invoke_rake_task("admin_dashboard:rebuild_rollups") }
+ end
+
+ it "rebuilds only the named rollup" do
+ DashboardRollupRebuilder.expects(:rebuild!).with("category_activity")
+
+ capture_stdout { invoke_rake_task("admin_dashboard:rebuild_rollups", "category_activity") }
+ end
+
+ it "aborts on an unknown rollup name" do
+ DashboardRollupRebuilder.expects(:rebuild!).never
+
+ expect { capture_stdout { invoke_rake_task("admin_dashboard:rebuild_rollups", "nope") } }.to(
+ raise_error(SystemExit),
+ )
+ end
+end