A platform for community discussion. Free, open, simple.
FEATURE: Exclude likely crawlers from dashboard metrics (#42611)
Krzysztof Kotlarek08e864298383
diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rbindex d173bb6e612..b399e4b68ee 100644--- a/app/controllers/search_controller.rb+++ b/app/controllers/search_controller.rb@@ -54,6 +54,7 @@ class SearchController < ApplicationControllersearch_args[:search_type] = :full_pagesearch_args[:ip_address] = request.remote_ipsearch_args[:user_agent] = request.user_agent+ search_args[:session_id] = pageview_session_idsearch_args[:user_id] = current_user.id if current_user.present?if rate_limit_search@@ -109,6 +110,7 @@ class SearchController < ApplicationControllersearch_args[:search_type] = :headersearch_args[:ip_address] = request.remote_ipsearch_args[:user_agent] = request.user_agent+ search_args[:session_id] = pageview_session_idsearch_args[:user_id] = current_user.id if current_user.present?search_args[:restrict_to_archetype] = params[:restrict_to_archetype] if params[:restrict_to_archetype@@ -164,6 +166,10 @@ class SearchController < ApplicationControllerprotected+ def pageview_session_id+ request.headers["HTTP_DISCOURSE_PAGEVIEW_SESSION_ID"]+ end+def site_overloaded?queue_time = request.env[Middleware::ProcessingRequest::REQUEST_QUEUE_SECONDS_ENV_KEY]if queue_timediff --git a/app/jobs/scheduled/detect_crawler_pageviews.rb b/app/jobs/scheduled/detect_crawler_pageviews.rbindex ec76c622999..7c9e2e15129 100644--- a/app/jobs/scheduled/detect_crawler_pageviews.rb+++ b/app/jobs/scheduled/detect_crawler_pageviews.rb@@ -8,10 +8,13 @@ module JobsWINDOW_DELAY = BrowserPageviewSessionEngagement::BEACON_SETTLE_PERIODdef execute(args)- return if !UpcomingChanges.enabled?(:improved_crawler_detection)+ return if !CrawlerScorer.enabled?window_end = Time.now - WINDOW_DELAY- CrawlerScorer.score!(window_start: window_end - LOOKBACK, window_end: window_end)+ window_start = window_end - LOOKBACK++ CrawlerScorer.score!(window_start: window_start, window_end: window_end)+ SearchLog.flag_likely_crawlers!(window_start: window_start, window_end: window_end)endendenddiff --git a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb b/app/jobs/scheduled/maintain_browser_pageview_rollups.rbindex 49ea0395527..14d604d38a4 100644--- a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb+++ b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb@@ -20,7 +20,7 @@ module Jobsprivatedef aggregate_crawlers- return if !UpcomingChanges.enabled?(:improved_crawler_detection)+ return if !CrawlerScorer.enabled?start_date, end_date = crawler_aggregation_windowreturn if start_date.nil?diff --git a/app/models/browser_pageview_country_daily_rollup.rb b/app/models/browser_pageview_country_daily_rollup.rbindex d7f3e7a29cb..7743fc50dd6 100644--- a/app/models/browser_pageview_country_daily_rollup.rb+++ b/app/models/browser_pageview_country_daily_rollup.rb@@ -6,12 +6,18 @@ class BrowserPageviewCountryDailyRollup < ActiveRecord::Baseend_date = end_date.to_date + 1DB.exec(<<~SQL, start_date:, end_date:)- INSERT INTO browser_pageview_country_daily_rollups (date, country_code, count, logged_in_count)+ INSERT INTO browser_pageview_country_daily_rollups (+ date, country_code, count, logged_in_count, likely_crawler_count, likely_crawler_logged_in_count+ )SELECTcreated_at::date AS date,country_code,COUNT(*) AS count,- COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS logged_in_count+ COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS logged_in_count,+ COUNT(*) FILTER (WHERE #{CrawlerScorer.likely_crawler_condition}) AS likely_crawler_count,+ COUNT(*) FILTER (+ WHERE user_id IS NOT NULL AND #{CrawlerScorer.likely_crawler_condition}+ ) AS likely_crawler_logged_in_countFROM browser_pageview_eventsWHERE created_at >= :start_dateAND created_at < :end_date@@ -19,7 +25,9 @@ class BrowserPageviewCountryDailyRollup < ActiveRecord::BaseGROUP BY date, country_codeON CONFLICT (date, country_code) DO UPDATESET count = EXCLUDED.count,- logged_in_count = EXCLUDED.logged_in_count+ logged_in_count = EXCLUDED.logged_in_count,+ likely_crawler_count = EXCLUDED.likely_crawler_count,+ likely_crawler_logged_in_count = EXCLUDED.likely_crawler_logged_in_countSQLendend@@ -28,11 +36,13 @@ end## Table name: browser_pageview_country_daily_rollups#-# id :bigint not null, primary key-# count :bigint not null-# country_code :string(2)-# date :date not null-# logged_in_count :bigint not null+# id :bigint not null, primary key+# count :bigint not null+# country_code :string(2)+# date :date not null+# likely_crawler_count :bigint default(0), not null+# likely_crawler_logged_in_count :bigint default(0), not null+# logged_in_count :bigint not null## Indexes#diff --git a/app/models/browser_pageview_event.rb b/app/models/browser_pageview_event.rbindex 2ba7fc58234..75c474b8122 100644--- a/app/models/browser_pageview_event.rb+++ b/app/models/browser_pageview_event.rb@@ -252,6 +252,15 @@ class BrowserPageviewEvent < ActiveRecord::Base)end+ def self.rollup_count_sql+ logged_in_only = SiteSetting.login_required+ count_column = logged_in_only ? "logged_in_count" : "count"+ return count_column if !CrawlerScorer.enabled?++ crawler_column = logged_in_only ? "likely_crawler_logged_in_count" : "likely_crawler_count"+ "GREATEST(#{count_column} - #{crawler_column}, 0)"+ end+before_save :truncate_fieldsprivatediff --git a/app/models/browser_pageview_referrer_daily_rollup.rb b/app/models/browser_pageview_referrer_daily_rollup.rbindex 5c8d1e3c51f..a72b6df10c6 100644--- a/app/models/browser_pageview_referrer_daily_rollup.rb+++ b/app/models/browser_pageview_referrer_daily_rollup.rb@@ -6,12 +6,18 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::Baseend_date = end_date.to_date + 1DB.exec(<<~SQL, start_date:, end_date:)- INSERT INTO browser_pageview_referrer_daily_rollups (date, normalized_referrer, count, logged_in_count)+ INSERT INTO browser_pageview_referrer_daily_rollups (+ date, normalized_referrer, count, logged_in_count, likely_crawler_count, likely_crawler_logged_in_count+ )SELECTcreated_at::date AS date,normalized_referrer,COUNT(*) AS count,- COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS logged_in_count+ COUNT(*) FILTER (WHERE user_id IS NOT NULL) AS logged_in_count,+ COUNT(*) FILTER (WHERE #{CrawlerScorer.likely_crawler_condition}) AS likely_crawler_count,+ COUNT(*) FILTER (+ WHERE user_id IS NOT NULL AND #{CrawlerScorer.likely_crawler_condition}+ ) AS likely_crawler_logged_in_countFROM browser_pageview_eventsWHERE created_at >= :start_dateAND created_at < :end_date@@ -19,7 +25,9 @@ class BrowserPageviewReferrerDailyRollup < ActiveRecord::BaseGROUP BY date, normalized_referrerON CONFLICT (date, normalized_referrer) DO UPDATESET count = EXCLUDED.count,- logged_in_count = EXCLUDED.logged_in_count+ logged_in_count = EXCLUDED.logged_in_count,+ likely_crawler_count = EXCLUDED.likely_crawler_count,+ likely_crawler_logged_in_count = EXCLUDED.likely_crawler_logged_in_countSQLend@@ -59,11 +67,13 @@ end## Table name: browser_pageview_referrer_daily_rollups#-# id :bigint not null, primary key-# count :bigint not null-# date :date not null-# logged_in_count :bigint not null-# normalized_referrer :string(2000)+# id :bigint not null, primary key+# count :bigint not null+# date :date not null+# likely_crawler_count :bigint default(0), not null+# likely_crawler_logged_in_count :bigint default(0), not null+# logged_in_count :bigint not null+# normalized_referrer :string(2000)## Indexes#diff --git a/app/models/browser_pageview_session_engagement_daily_rollup.rb b/app/models/browser_pageview_session_engagement_daily_rollup.rbindex 78ab6274721..c4b2bf49406 100644--- a/app/models/browser_pageview_session_engagement_daily_rollup.rb+++ b/app/models/browser_pageview_session_engagement_daily_rollup.rb@@ -41,7 +41,8 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::Basebpe.session_id,MIN(bpe.created_at)::date AS date,COUNT(*) AS pageview_count,- bool_or(bpe.user_id IS NOT NULL) AS logged_in+ bool_or(bpe.user_id IS NOT NULL) AS logged_in,+ bool_or(#{CrawlerScorer.likely_crawler_condition(table: "bpe")}) AS likely_crawlerFROM browser_pageview_events bpeJOIN active_sessions ON active_sessions.session_id = bpe.session_idWHERE #{BrowserPageviewEvent.rollup_source_condition(table: "bpe")}@@ -49,7 +50,10 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::BaseHAVING MIN(bpe.created_at) >= :start_date)INSERT INTO browser_pageview_session_engagement_daily_rollups- (date, logged_in, sessions, bounced, engaged_seconds_total)+ (+ date, logged_in, sessions, bounced, engaged_seconds_total,+ likely_crawler_sessions, likely_crawler_bounced, likely_crawler_engaged_seconds_total+ )SELECTsession_pageviews.date,session_pageviews.logged_in,@@ -58,7 +62,17 @@ class BrowserPageviewSessionEngagementDailyRollup < ActiveRecord::BaseWHERE session_pageviews.pageview_count = 1AND COALESCE(engagement.engaged_seconds, 0) < :bounce_threshold) AS bounced,- COALESCE(SUM(engagement.engaged_seconds), 0) AS engaged_seconds_total+ COALESCE(SUM(engagement.engaged_seconds), 0) AS engaged_seconds_total,+ COUNT(*) FILTER (WHERE session_pageviews.likely_crawler) AS likely_crawler_sessions,+ COUNT(*) FILTER (+ WHERE session_pageviews.likely_crawler+ AND session_pageviews.pageview_count = 1+ AND COALESCE(engagement.engaged_seconds, 0) < :bounce_threshold+ ) AS likely_crawler_bounced,+ COALESCE(+ SUM(engagement.engaged_seconds) FILTER (WHERE session_pageviews.likely_crawler),+ 0+ ) AS likely_crawler_engaged_seconds_totalFROM session_pageviewsLEFT JOIN browser_pageview_session_engagements engagementON engagement.session_id = session_pageviews.session_id@@ -77,12 +91,15 @@ end## Table name: browser_pageview_session_engagement_daily_rollups#-# id :bigint not null, primary key-# bounced :bigint not null-# date :date not null-# engaged_seconds_total :bigint not null-# logged_in :boolean not null-# sessions :bigint not null+# id :bigint not null, primary key+# bounced :bigint not null+# date :date not null+# engaged_seconds_total :bigint not null+# likely_crawler_bounced :bigint default(0), not null+# likely_crawler_engaged_seconds_total :bigint default(0), not null+# likely_crawler_sessions :bigint default(0), not null+# logged_in :boolean not null+# sessions :bigint not null## Indexes#diff --git a/app/models/category_activity_daily_rollup.rb b/app/models/category_activity_daily_rollup.rbindex 8d15f46701f..82536ae1fbe 100644--- a/app/models/category_activity_daily_rollup.rb+++ b/app/models/category_activity_daily_rollup.rb@@ -20,6 +20,8 @@ class CategoryActivityDailyRollup < ActiveRecord::Basecategory_ids: nil,secure_category_ids: nil)+ page_views = human_page_views_expr+builder = DB.build(<<~SQL)SELECTc.id,@@ -28,15 +30,15 @@ class CategoryActivityDailyRollup < ActiveRecord::Basec.slug,COALESCE(SUM(r.topics) FILTER (WHERE r.date >= :current_start), 0)::bigint AS topics_current,COALESCE(SUM(r.posts) FILTER (WHERE r.date >= :current_start), 0)::bigint AS posts_current,- COALESCE(SUM(r.page_views) FILTER (WHERE r.date >= :current_start), 0)::bigint AS page_views_current,+ COALESCE(SUM(#{page_views}) FILTER (WHERE r.date >= :current_start), 0)::bigint AS page_views_current,COALESCE(SUM(r.topics) FILTER (WHERE r.date < :current_start), 0)::bigint AS topics_prior,COALESCE(SUM(r.posts) FILTER (WHERE r.date < :current_start), 0)::bigint AS posts_prior,- COALESCE(SUM(r.page_views) FILTER (WHERE r.date < :current_start), 0)::bigint AS page_views_prior+ COALESCE(SUM(#{page_views}) FILTER (WHERE r.date < :current_start), 0)::bigint AS page_views_priorFROM category_activity_daily_rollups rINNER JOIN categories c ON c.id = r.category_id/*where*/GROUP BY c.id, c.name, c.color, c.slug- HAVING SUM(r.topics + r.posts + r.page_views) > 0+ HAVING SUM(r.topics + r.posts + #{page_views}) > 0SQLbuilder.where("r.date >= :prev_start AND r.date <= :current_end")@@ -46,6 +48,13 @@ class CategoryActivityDailyRollup < ActiveRecord::Basebuilder.query(prev_start: prev_start, current_start: current_start, current_end: current_end)end+ def self.human_page_views_expr+ return "r.page_views" if !CrawlerScorer.enabled?++ "GREATEST(r.page_views - r.likely_crawler_page_views, 0)"+ end+ private_class_method :human_page_views_expr+def self.earliest_activity_dateTopic.where(ELIGIBLE_TOPICS).minimum(:created_at)&.to_dateend@@ -105,12 +114,29 @@ class CategoryActivityDailyRollup < ActiveRecord::BaseAND tvs.viewed_at < :end_dateAND #{ELIGIBLE_TOPICS}GROUP BY 1, 2+ ), crawler_view_counts AS (+ SELECT bpe.created_at::date AS date,+ topics.category_id,+ COUNT(DISTINCT (+ bpe.topic_id,+ COALESCE(bpe.user_id::text, 'ip-' || bpe.ip_address::text)+ )) AS likely_crawler_page_views+ FROM browser_pageview_events bpe+ INNER JOIN topics ON topics.id = bpe.topic_id+ WHERE bpe.created_at >= :start_date+ AND bpe.created_at < :end_date+ AND #{CrawlerScorer.likely_crawler_condition(table: "bpe")}+ AND #{BrowserPageviewEvent.rollup_source_condition(table: "bpe")}+ AND #{ELIGIBLE_TOPICS}+ GROUP BY 1, 2)- SELECT date,- category_id,- SUM(topics)::int AS topics,- SUM(posts)::int AS posts,- SUM(page_views)::bigint AS page_views+ SELECT combined.date,+ combined.category_id,+ SUM(combined.topics)::int AS topics,+ SUM(combined.posts)::int AS posts,+ SUM(combined.page_views)::bigint AS page_views,+ COALESCE(MAX(crawler_view_counts.likely_crawler_page_views), 0)::bigint+ AS likely_crawler_page_viewsFROM (SELECT date, category_id, topics, 0 AS posts, 0 AS page_views FROM topic_countsUNION ALL@@ -118,7 +144,10 @@ class CategoryActivityDailyRollup < ActiveRecord::BaseUNION ALLSELECT date, category_id, 0, 0, page_views FROM view_counts) combined- GROUP BY date, category_id+ LEFT JOIN crawler_view_counts+ ON crawler_view_counts.date = combined.date+ AND crawler_view_counts.category_id = combined.category_id+ GROUP BY combined.date, combined.category_idSQLstart_date: start_date,end_date: end_date + 1.day,@@ -128,8 +157,13 @@ class CategoryActivityDailyRollup < ActiveRecord::Baseprivate_class_method :daily_countsdef self.replace!(start_date:, end_date:, rows:)+ if rows.empty?+ where(date: start_date..end_date).delete_all+ return+ end++ preserved = crawler_page_views_without_events(start_date, end_date)where(date: start_date..end_date).delete_all- return if rows.empty?insert_all!(rows.map do |row|@@ -139,23 +173,45 @@ class CategoryActivityDailyRollup < ActiveRecord::Basetopics: row.topics,posts: row.posts,page_views: row.page_views,+ likely_crawler_page_views:+ preserved.fetch([row.date, row.category_id], row.likely_crawler_page_views),}end,)endprivate_class_method :replace!++ def self.crawler_page_views_without_events(start_date, end_date)+ dates_with_events =+ DB.query_single(<<~SQL, start_date: start_date, end_date: end_date.to_date + 1)+ SELECT DISTINCT created_at::date+ FROM browser_pageview_events+ WHERE created_at >= :start_date+ AND created_at < :end_date+ AND #{BrowserPageviewEvent.rollup_source_condition}+ SQL++ scope = where(date: start_date..end_date).where("likely_crawler_page_views > 0")+ scope = scope.where.not(date: dates_with_events) if dates_with_events.present?++ scope+ .pluck(:date, :category_id, :likely_crawler_page_views)+ .to_h { |date, category_id, count| [[date, category_id], count] }+ end+ private_class_method :crawler_page_views_without_eventsend# == Schema Information## Table name: category_activity_daily_rollups#-# id :bigint not null, primary key-# date :date not null-# page_views :bigint default(0), not null-# posts :integer default(0), not null-# topics :integer default(0), not null-# category_id :integer not null+# id :bigint not null, primary key+# date :date not null+# likely_crawler_page_views :bigint default(0), not null+# page_views :bigint default(0), not null+# posts :integer default(0), not null+# topics :integer default(0), not null+# category_id :integer not null## Indexes#diff --git a/app/models/concerns/reports/site_traffic.rb b/app/models/concerns/reports/site_traffic.rbindex da003046c8d..30519dc9078 100644--- a/app/models/concerns/reports/site_traffic.rb+++ b/app/models/concerns/reports/site_traffic.rb@@ -26,7 +26,7 @@ module Reports::SiteTrafficpage_view_anon_browser: ApplicationRequest.req_types[:page_view_anon_browser],).first- likely_crawlers_enabled = UpcomingChanges.enabled?(:improved_crawler_detection)+ likely_crawlers_enabled = CrawlerScorer.enabled?data =DB.query(diff --git a/app/models/concerns/reports/top_countries_by_browser_pageviews.rb b/app/models/concerns/reports/top_countries_by_browser_pageviews.rbindex c6efe492a99..7b173ffe4dd 100644--- a/app/models/concerns/reports/top_countries_by_browser_pageviews.rb+++ b/app/models/concerns/reports/top_countries_by_browser_pageviews.rb@@ -23,20 +23,20 @@ module Reports::TopCountriesByBrowserPageviews},]- count_expr = SiteSetting.login_required ? "logged_in_count" : "count"+ count_sql = BrowserPageviewEvent.rollup_count_sqlend_date_exclusive = report.end_date.to_date + 1sql = <<~SQLWITH ranked AS (SELECTcountry_code,- SUM(#{count_expr}) AS count,- SUM(SUM(#{count_expr})) OVER () AS total+ SUM(#{count_sql}) AS count,+ SUM(SUM(#{count_sql})) OVER () AS totalFROM browser_pageview_country_daily_rollupsWHERE date >= :start_dateAND date < :end_date_exclusiveGROUP BY country_code- HAVING SUM(#{count_expr}) > 0+ HAVING SUM(#{count_sql}) > 0)SELECT country_code, count,CASE WHEN total = 0 THEN 0diff --git a/app/models/concerns/reports/top_referrers_by_browser_pageviews.rb b/app/models/concerns/reports/top_referrers_by_browser_pageviews.rbindex 18c4fba4c80..44518e6bc8b 100644--- a/app/models/concerns/reports/top_referrers_by_browser_pageviews.rb+++ b/app/models/concerns/reports/top_referrers_by_browser_pageviews.rb@@ -22,7 +22,7 @@ module Reports::TopReferrersByBrowserPageviews},]- count_expr = SiteSetting.login_required ? "logged_in_count" : "count"+ count_sql = BrowserPageviewEvent.rollup_count_sqlend_date_exclusive = report.end_date.to_date + 1host = BrowserPageviewEventUrlNormalizer.normalize_host(Discourse.current_hostname)@@ -32,8 +32,8 @@ module Reports::TopReferrersByBrowserPageviewsWITH ranked AS (SELECTnormalized_referrer,- SUM(#{count_expr}) AS count,- SUM(SUM(#{count_expr})) OVER () AS total+ SUM(#{count_sql}) AS count,+ SUM(SUM(#{count_sql})) OVER () AS totalFROM browser_pageview_referrer_daily_rollupsWHERE date >= :start_dateAND date < :end_date_exclusive@@ -42,7 +42,7 @@ module Reports::TopReferrersByBrowserPageviewsAND normalized_referrer NOT LIKE :host_path_prefix ESCAPE '\\'AND normalized_referrer NOT LIKE :host_query_prefix ESCAPE '\\'GROUP BY normalized_referrer- HAVING SUM(#{count_expr}) > 0+ HAVING SUM(#{count_sql}) > 0)SELECT normalized_referrer, count,CASE WHEN total = 0 THEN 0diff --git a/app/models/report.rb b/app/models/report.rbindex 8c18e1a2c89..4f70e5ded66 100644--- a/app/models/report.rb+++ b/app/models/report.rb@@ -220,6 +220,7 @@ class ReportSCHEMA_VERSION,guardian&.user&.id || report.current_user&.id,guardian&.can_see_ip?,+ CrawlerScorer.enabled?,].compact.map(&:to_s).join(":")enddiff --git a/app/models/search_log.rb b/app/models/search_log.rbindex f003a1814cd..e0040597222 100644--- a/app/models/search_log.rb+++ b/app/models/search_log.rb@@ -2,6 +2,7 @@class SearchLog < ActiveRecord::BaseMAXIMUM_USER_AGENT_LENGTH = 2000+ MAXIMUM_SESSION_ID_LENGTH = BrowserPageviewEvent::MAX_SESSION_ID_LENGTHvalidates :term, presence: truevalidates :user_agent, length: { maximum: MAXIMUM_USER_AGENT_LENGTH }@@ -9,6 +10,15 @@ class SearchLog < ActiveRecord::Basebelongs_to :userscope :non_staff, -> { joins(:user).where(users: { admin: false, moderator: false }) }+ scope :non_staff_or_anonymous,+ -> do+ left_outer_joins(:user).where(+ "users.id IS NULL OR (NOT users.admin AND NOT users.moderator)",+ )+ end+ scope :excluding_crawlers, -> { where(crawler: false, likely_crawler: false) }+ scope :human_only,+ -> { CrawlerScorer.enabled? ? non_staff_or_anonymous.excluding_crawlers : non_staff }def ctrreturn 0 if click_through == 0 || searches == 0@@ -37,7 +47,7 @@ class SearchLog < ActiveRecord::BaseDiscourse.redis.keys("__SEARCH__LOG_*").each { |k| Discourse.redis.del(k) }end- def self.log(term:, search_type:, ip_address:, user_agent: nil, user_id: nil)+ def self.log(term:, search_type:, ip_address:, user_agent: nil, user_id: nil, session_id: nil)return [:error] if term.blank?can_log_search =@@ -74,6 +84,8 @@ class SearchLog < ActiveRecord::Baseip_address: ip_address,user_agent: user_agent,user_id: user_id,+ session_id: session_id&.slice(0, MAXIMUM_SESSION_ID_LENGTH),+ crawler: user_agent.present? && CrawlerDetection.crawler?(user_agent),)result = [:created, log.id]@@ -84,6 +96,47 @@ class SearchLog < ActiveRecord::Baseresultend+ BACKFILL_AGENT_BATCH_SIZE = 100++ def self.backfill_crawler!+ agents = DB.query_single(<<~SQL)+ SELECT DISTINCT user_agent+ FROM search_logs+ WHERE NOT crawler+ AND user_agent IS NOT NULL+ AND user_agent <> ''+ SQL++ crawler_agents = agents.select { |agent| CrawlerDetection.crawler?(agent) }+ return 0 if crawler_agents.empty?++ crawler_agents+ .each_slice(BACKFILL_AGENT_BATCH_SIZE)+ .sum { |batch| DB.exec(<<~SQL, agents: batch) }+ UPDATE search_logs+ SET crawler = TRUE+ WHERE NOT crawler+ AND user_agent IN (:agents)+ SQL+ end++ def self.flag_likely_crawlers!(window_start:, window_end:)+ DB.exec(<<~SQL, window_start: window_start, window_end: window_end)+ UPDATE search_logs+ SET likely_crawler = TRUE+ WHERE NOT search_logs.likely_crawler+ AND search_logs.session_id IS NOT NULL+ AND search_logs.created_at >= :window_start+ AND search_logs.created_at < :window_end+ AND EXISTS (+ SELECT 1+ FROM browser_pageview_events event+ WHERE event.session_id = search_logs.session_id+ AND #{CrawlerScorer.likely_crawler_condition(table: "event")}+ )+ SQL+ end+def self.term_details(term, period = :weekly, search_type = :all)details = []@@ -98,6 +151,7 @@ class SearchLog < ActiveRecord::Basesearch_type == :full_pageresult = result.where.not(search_result_id: nil) if search_type == :click_through_onlyresult = result.non_staff if search_type == :non_staff_only+ result = result.human_only if search_type == :human_onlyresult.order("date")@@ -138,6 +192,8 @@ class SearchLog < ActiveRecord::Baseif search_type == :non_staff_onlyresult = result.non_staff+ elsif search_type == :human_only+ result = result.human_onlyelsif search_type != :allresult = result.where("search_type = ?", search_types[search_type])end@@ -182,17 +238,21 @@ end# Table name: search_logs## id :integer not null, primary key+# crawler :boolean default(FALSE), not null# ip_address :inet+# likely_crawler :boolean default(FALSE), not null# search_result_type :integer# search_type :integer not null# term :string not null# user_agent :string(2000)# created_at :datetime not null# search_result_id :integer+# session_id :string(32)# user_id :integer## Indexes#-# index_search_logs_on_created_at (created_at)-# index_search_logs_on_user_id_and_created_at (user_id,created_at) WHERE (user_id IS NOT NULL)+# index_search_logs_on_created_at (created_at)+# index_search_logs_on_created_at_excluding_crawlers (created_at) WHERE ((NOT crawler) AND (NOT likely_crawler))+# index_search_logs_on_user_id_and_created_at (user_id,created_at) WHERE (user_id IS NOT NULL)#diff --git a/app/services/admin_dashboard_search.rb b/app/services/admin_dashboard_search.rbindex f444e4b5343..86fbab56d90 100644--- a/app/services/admin_dashboard_search.rb+++ b/app/services/admin_dashboard_search.rb@@ -44,6 +44,7 @@ class AdminDashboardSearchtrending: trending,trending_period: trending_period,content_gaps: content_gaps,+ search_type: search_type,}end@@ -100,7 +101,7 @@ class AdminDashboardSearchdef trendingrows =- non_staff_search_logs_in(window_start: start_date, window_end: end_date)+ human_search_logs_in(window_start: start_date, window_end: end_date).select(<<~SQL)lower(search_logs.term) AS term,COUNT(*) AS searches,@@ -123,7 +124,7 @@ class AdminDashboardSearchdef content_gapsrows =- non_staff_search_logs_in(window_start: start_date, window_end: end_date)+ human_search_logs_in(window_start: start_date, window_end: end_date).select(<<~SQL)lower(search_logs.term) AS term,COUNT(*) AS searches,@@ -148,12 +149,12 @@ class AdminDashboardSearchdef window_stats(window_start:, window_end:)term_stats =- non_staff_search_logs_in(window_start: window_start, window_end: window_end).select(- <<~SQL,+ human_search_logs_in(window_start: window_start, window_end: window_end).select(<<~SQL).group(COUNT(*) AS searches,SUM(CASE WHEN search_result_id IS NOT NULL THEN 1 ELSE 0 END) AS clicksSQL- ).group("lower(search_logs.term)")+ "lower(search_logs.term)",+ )row = SearchLog.from("(#{term_stats.to_sql}) term_stats").select(<<~SQL).takeCOALESCE(SUM(searches), 0)::bigint AS total,@@ -163,8 +164,12 @@ class AdminDashboardSearch{ total: row.total, no_match: row.no_match }end- def non_staff_search_logs_in(window_start:, window_end:)- SearchLog.non_staff.where(created_at: window_start..window_end)+ def human_search_logs_in(window_start:, window_end:)+ SearchLog.human_only.where(created_at: window_start..window_end)+ end++ def search_type+ CrawlerScorer.enabled? ? "human_only" : "non_staff_only"enddef parse_date(value)diff --git a/app/services/admin_dashboard_site_traffic.rb b/app/services/admin_dashboard_site_traffic.rbindex 04b34a0a19b..cb5bb654e87 100644--- a/app/services/admin_dashboard_site_traffic.rb+++ b/app/services/admin_dashboard_site_traffic.rb@@ -105,7 +105,7 @@ class AdminDashboardSiteTrafficdef likely_crawlers_enabled?return @likely_crawlers_enabled if defined?(@likely_crawlers_enabled)- @likely_crawlers_enabled = UpcomingChanges.enabled?(:improved_crawler_detection)+ @likely_crawlers_enabled = CrawlerScorer.enabled?enddef kpis(totals, prior_rows)@@ -144,9 +144,12 @@ class AdminDashboardSiteTrafficDB.query_hash(<<~SQL, start_date: start_date.to_date, end_date: end_date.to_date)SELECT- COALESCE(SUM(sessions), 0)::bigint AS sessions,- COALESCE(SUM(bounced), 0)::bigint AS bounced,- COALESCE(SUM(engaged_seconds_total), 0)::bigint AS engaged_seconds_total+ COALESCE(SUM(#{human_column("sessions", "likely_crawler_sessions")}), 0)::bigint AS sessions,+ COALESCE(SUM(#{human_column("bounced", "likely_crawler_bounced")}), 0)::bigint AS bounced,+ COALESCE(+ SUM(#{human_column("engaged_seconds_total", "likely_crawler_engaged_seconds_total")}),+ 0+ )::bigint AS engaged_seconds_totalFROM browser_pageview_session_engagement_daily_rollupsWHERE date >= :start_dateAND date <= :end_date@@ -155,6 +158,12 @@ class AdminDashboardSiteTraffic.symbolize_keysend+ def human_column(column, crawler_column)+ return column if !likely_crawlers_enabled?++ "GREATEST(#{column} - #{crawler_column}, 0)"+ end+def browser_pageviews_kpi(totals, prior_rows)kpi = { value: totals[:human] }trend = build_trend(totals, prior_rows)@@ -173,12 +182,12 @@ class AdminDashboardSiteTrafficdef direct_traffic_valuereturn nil if !SiteSetting.persist_browser_pageview_events- count_column = login_required? ? "logged_in_count" : "count"+ count_sql = BrowserPageviewEvent.rollup_count_sqlrow = DB.query(<<~SQL, start_date: start_date.to_date, end_date: end_date.to_date).firstSELECT- COALESCE(SUM(#{count_column}), 0)::bigint AS total,- COALESCE(SUM(#{count_column}) FILTER (WHERE normalized_referrer IS NULL), 0)::bigint AS direct+ COALESCE(SUM(#{count_sql}), 0)::bigint AS total,+ COALESCE(SUM(#{count_sql}) FILTER (WHERE normalized_referrer IS NULL), 0)::bigint AS directFROM browser_pageview_referrer_daily_rollupsWHERE date >= :start_dateAND date <= :end_datediff --git a/app/services/admin_dashboard_site_traffic_explorer.rb b/app/services/admin_dashboard_site_traffic_explorer.rbindex 699b5a89649..47c79f533e6 100644--- a/app/services/admin_dashboard_site_traffic_explorer.rb+++ b/app/services/admin_dashboard_site_traffic_explorer.rb@@ -155,7 +155,7 @@ class AdminDashboardSiteTrafficExplorerinclude_logged_in: filters[:traffic_type]&.include?("logged_in"),include_anonymous: filters[:traffic_type]&.include?("anonymous"),include_likely_crawler: filters[:traffic_type]&.include?("likely_crawler"),- crawler_detection_enabled: UpcomingChanges.enabled?(:improved_crawler_detection),+ crawler_detection_enabled: CrawlerScorer.enabled?,crawler_threshold: CrawlerScorer::BOT_SCORE_THRESHOLD,bounce_threshold:BrowserPageviewSessionEngagementDailyRollup.bounce_engaged_seconds_threshold,diff --git a/config/locales/client.en.yml b/config/locales/client.en.ymlindex 0551883db07..772af7d26af 100644--- a/config/locales/client.en.yml+++ b/config/locales/client.en.yml@@ -7574,7 +7574,7 @@ en:kpi:total_searches:label: "Total searches"- tooltip: "The number of searches performed by members in your community."+ tooltip: "The number of searches performed in your community."no_result_rate:label: "No-result rate"tooltip: "The percentage of searches where members didn't click any result (0% CTR). This indicates that members may be searching for content your community doesn't have."@@ -7584,7 +7584,7 @@ en:status: "Status"trending:title: "Trending searches"- tooltip: "The most popular search terms among members."+ tooltip: "The most popular search terms in your community."empty: "No searches in this period."content_gaps:title: "Content gaps"@@ -9223,6 +9223,7 @@ en:full_page: "Full page"click_through_only: "All (click through only)"non_staff_only: "Non-staff users"+ human_only: "Excluding staff and crawlers"header_search_results: "Header search results"logster:title: "Error logs"diff --git a/db/migrate/20260813071223_add_likely_crawler_columns.rb b/db/migrate/20260813071223_add_likely_crawler_columns.rbnew file mode 100644index 00000000000..d0e4803bb8e--- /dev/null+++ b/db/migrate/20260813071223_add_likely_crawler_columns.rb@@ -0,0 +1,53 @@+# frozen_string_literal: true++class AddLikelyCrawlerColumns < ActiveRecord::Migration[8.0]+ def change+ add_column :browser_pageview_country_daily_rollups,+ :likely_crawler_count,+ :bigint,+ null: false,+ default: 0+ add_column :browser_pageview_country_daily_rollups,+ :likely_crawler_logged_in_count,+ :bigint,+ null: false,+ default: 0++ add_column :browser_pageview_referrer_daily_rollups,+ :likely_crawler_count,+ :bigint,+ null: false,+ default: 0+ add_column :browser_pageview_referrer_daily_rollups,+ :likely_crawler_logged_in_count,+ :bigint,+ null: false,+ default: 0++ add_column :browser_pageview_session_engagement_daily_rollups,+ :likely_crawler_sessions,+ :bigint,+ null: false,+ default: 0+ add_column :browser_pageview_session_engagement_daily_rollups,+ :likely_crawler_bounced,+ :bigint,+ null: false,+ default: 0+ add_column :browser_pageview_session_engagement_daily_rollups,+ :likely_crawler_engaged_seconds_total,+ :bigint,+ null: false,+ default: 0++ add_column :category_activity_daily_rollups,+ :likely_crawler_page_views,+ :bigint,+ null: false,+ default: 0++ add_column :search_logs, :crawler, :boolean, null: false, default: false+ add_column :search_logs, :likely_crawler, :boolean, null: false, default: false+ add_column :search_logs, :session_id, :string, limit: 32+ end+enddiff --git a/db/migrate/20260813071230_add_search_logs_created_at_excluding_crawlers_index.rb b/db/migrate/20260813071230_add_search_logs_created_at_excluding_crawlers_index.rbnew file mode 100644index 00000000000..6679f334a9c--- /dev/null+++ b/db/migrate/20260813071230_add_search_logs_created_at_excluding_crawlers_index.rb@@ -0,0 +1,20 @@+# frozen_string_literal: true++class AddSearchLogsCreatedAtExcludingCrawlersIndex < ActiveRecord::Migration[8.0]+ disable_ddl_transaction!++ INDEX_NAME = "index_search_logs_on_created_at_excluding_crawlers"++ def up+ remove_index :search_logs, name: INDEX_NAME, algorithm: :concurrently, if_exists: true+ add_index :search_logs,+ :created_at,+ name: INDEX_NAME,+ where: "NOT crawler AND NOT likely_crawler",+ algorithm: :concurrently+ end++ def down+ remove_index :search_logs, name: INDEX_NAME, algorithm: :concurrently, if_exists: true+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex 1eb5810955e..290c6592911 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -1957,7 +1957,9 @@ CREATE TABLE public.browser_pageview_country_daily_rollups (date date NOT NULL,country_code character varying(2),count bigint NOT NULL,- logged_in_count bigint NOT NULL+ logged_in_count bigint NOT NULL,+ likely_crawler_count bigint DEFAULT 0 NOT NULL,+ likely_crawler_logged_in_count bigint DEFAULT 0 NOT NULL);@@ -2105,7 +2107,9 @@ CREATE TABLE public.browser_pageview_referrer_daily_rollups (date date NOT NULL,normalized_referrer character varying(2000),count bigint NOT NULL,- logged_in_count bigint NOT NULL+ logged_in_count bigint NOT NULL,+ likely_crawler_count bigint DEFAULT 0 NOT NULL,+ likely_crawler_logged_in_count bigint DEFAULT 0 NOT NULL);@@ -2138,7 +2142,10 @@ CREATE TABLE public.browser_pageview_session_engagement_daily_rollups (logged_in boolean NOT NULL,sessions bigint NOT NULL,bounced bigint NOT NULL,- engaged_seconds_total bigint NOT NULL+ engaged_seconds_total bigint NOT NULL,+ likely_crawler_sessions bigint DEFAULT 0 NOT NULL,+ likely_crawler_bounced bigint DEFAULT 0 NOT NULL,+ likely_crawler_engaged_seconds_total bigint DEFAULT 0 NOT NULL);@@ -2281,7 +2288,8 @@ CREATE TABLE public.category_activity_daily_rollups (category_id integer NOT NULL,topics integer DEFAULT 0 NOT NULL,posts integer DEFAULT 0 NOT NULL,- page_views bigint DEFAULT 0 NOT NULL+ page_views bigint DEFAULT 0 NOT NULL,+ likely_crawler_page_views bigint DEFAULT 0 NOT NULL);@@ -8803,7 +8811,10 @@ CREATE TABLE public.search_logs (search_type integer NOT NULL,created_at timestamp without time zone NOT NULL,search_result_type integer,- user_agent character varying(2000)+ user_agent character varying(2000),+ crawler boolean DEFAULT false NOT NULL,+ likely_crawler boolean DEFAULT false NOT NULL,+ session_id character varying(32));@@ -21273,6 +21284,13 @@ CREATE UNIQUE INDEX index_screened_urls_on_url ON public.screened_urls USING btrCREATE INDEX index_search_logs_on_created_at ON public.search_logs USING btree (created_at);+--+-- Name: index_search_logs_on_created_at_excluding_crawlers; Type: INDEX; Schema: public; Owner: -+--++CREATE INDEX index_search_logs_on_created_at_excluding_crawlers ON public.search_logs USING btree (created_at) WHERE ((NOT crawler) AND (NOT likely_crawler));++---- Name: index_search_logs_on_user_id_and_created_at; Type: INDEX; Schema: public; Owner: ---@@ -23178,6 +23196,8 @@ INSERT INTO "schema_migrations" (version) VALUES('20260814060100'),('20260814060000'),('20260813202233'),+('20260813071230'),+('20260813071223'),('20260812094609'),('20260811231259'),('20260810154331'),diff --git a/frontend/discourse/admin/components/dashboard/search.gjs b/frontend/discourse/admin/components/dashboard/search.gjsindex a205057fcf7..d45a46873df 100644--- a/frontend/discourse/admin/components/dashboard/search.gjs+++ b/frontend/discourse/admin/components/dashboard/search.gjs@@ -273,7 +273,7 @@ export default class DashboardSearch extends Component {@query={{hashterm=row.termperiod=this.trendingTermPeriod- searchType="non_staff_only"+ [email protected]_type}}title={{row.term}}>diff --git a/frontend/discourse/admin/controllers/admin-search-logs/index.js b/frontend/discourse/admin/controllers/admin-search-logs/index.jsindex 8cbb815ee2d..c1eccaa7eb8 100644--- a/frontend/discourse/admin/controllers/admin-search-logs/index.js+++ b/frontend/discourse/admin/controllers/admin-search-logs/index.js@@ -1,25 +1,39 @@import Controller from "@ember/controller";+import { service } from "@ember/service";import { i18n } from "discourse-i18n";export const DEFAULT_PERIOD = "yearly";export default class AdminSearchLogsIndexController extends Controller {+ @service siteSettings;+loading = false;period = DEFAULT_PERIOD;searchType = "all";- searchTypeOptions = [- {- id: "all",- name: i18n("admin.logs.search_logs.types.all_search_types"),- },- {- id: "non_staff_only",- name: i18n("admin.logs.search_logs.types.non_staff_only"),- },- { id: "header", name: i18n("admin.logs.search_logs.types.header") },- {- id: "full_page",- name: i18n("admin.logs.search_logs.types.full_page"),- },- ];++ get searchTypeOptions() {+ return [+ {+ id: "all",+ name: i18n("admin.logs.search_logs.types.all_search_types"),+ },+ {+ id: "non_staff_only",+ name: i18n("admin.logs.search_logs.types.non_staff_only"),+ },+ ...(this.siteSettings.improved_crawler_detection+ ? [+ {+ id: "human_only",+ name: i18n("admin.logs.search_logs.types.human_only"),+ },+ ]+ : []),+ { id: "header", name: i18n("admin.logs.search_logs.types.header") },+ {+ id: "full_page",+ name: i18n("admin.logs.search_logs.types.full_page"),+ },+ ];+ }}diff --git a/frontend/discourse/admin/controllers/admin-search-logs/term.js b/frontend/discourse/admin/controllers/admin-search-logs/term.jsindex c37c4d0c2d1..4958044a60f 100644--- a/frontend/discourse/admin/controllers/admin-search-logs/term.js+++ b/frontend/discourse/admin/controllers/admin-search-logs/term.js@@ -1,31 +1,45 @@import Controller from "@ember/controller";+import { service } from "@ember/service";import { DEFAULT_PERIOD } from "discourse/admin/controllers/admin-search-logs/index";import { i18n } from "discourse-i18n";export default class AdminSearchLogsTermController extends Controller {+ @service siteSettings;+loading = false;term = null;period = DEFAULT_PERIOD;searchType = "all";- searchTypeOptions = [- {- id: "all",- name: i18n("admin.logs.search_logs.types.all_search_types"),- },- {- id: "non_staff_only",- name: i18n("admin.logs.search_logs.types.non_staff_only"),- },- { id: "header", name: i18n("admin.logs.search_logs.types.header") },- {- id: "full_page",- name: i18n("admin.logs.search_logs.types.full_page"),- },- {- id: "click_through_only",- name: i18n("admin.logs.search_logs.types.click_through_only"),- },- ];++ get searchTypeOptions() {+ return [+ {+ id: "all",+ name: i18n("admin.logs.search_logs.types.all_search_types"),+ },+ {+ id: "non_staff_only",+ name: i18n("admin.logs.search_logs.types.non_staff_only"),+ },+ ...(this.siteSettings.improved_crawler_detection+ ? [+ {+ id: "human_only",+ name: i18n("admin.logs.search_logs.types.human_only"),+ },+ ]+ : []),+ { id: "header", name: i18n("admin.logs.search_logs.types.header") },+ {+ id: "full_page",+ name: i18n("admin.logs.search_logs.types.full_page"),+ },+ {+ id: "click_through_only",+ name: i18n("admin.logs.search_logs.types.click_through_only"),+ },+ ];+ }get chartConfig() {return {diff --git a/frontend/discourse/app/controllers/full-page-search.js b/frontend/discourse/app/controllers/full-page-search.jsindex 4f36cfacde9..bfc3011aed4 100644--- a/frontend/discourse/app/controllers/full-page-search.js+++ b/frontend/discourse/app/controllers/full-page-search.js@@ -479,7 +479,16 @@ export default class FullPageSearchController extends Controller {if (this.currentUser) {updateRecentSearches(this.currentUser, searchTerm);}- ajax("/search", { data: args })+ const sessionId = document.querySelector(+ "meta[name=discourse-track-view-session-id]"+ )?.content;++ ajax("/search", {+ data: args,+ headers: sessionId+ ? { "Discourse-Pageview-Session-Id": sessionId }+ : {},+ }).then(async (results) => {const model = (await translateResults(results)) || {};diff --git a/frontend/discourse/app/lib/search.js b/frontend/discourse/app/lib/search.jsindex 27747ae5fbd..07499241cd6 100644--- a/frontend/discourse/app/lib/search.js+++ b/frontend/discourse/app/lib/search.js@@ -189,7 +189,14 @@ export function searchForTerm(term, opts) {};}- let ajaxPromise = ajax("/search/query", { data });+ const sessionId = document.querySelector(+ "meta[name=discourse-track-view-session-id]"+ )?.content;+ const headers = sessionId+ ? { "Discourse-Pageview-Session-Id": sessionId }+ : {};++ let ajaxPromise = ajax("/search/query", { data, headers });const promise = ajaxPromise.then((res) => translateResults(res, opts));promise.abort = ajaxPromise.abort;return promise;diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rbindex 2cc0dc0165f..1fb50682ec0 100644--- a/lib/crawler_scorer.rb+++ b/lib/crawler_scorer.rb@@ -44,6 +44,15 @@ class CrawlerScorerMISSING_ENGAGEMENT_HIGH_SCORE = 40ENGAGEMENT_LOOKBACK = 6.hours+ def self.enabled?+ UpcomingChanges.enabled?(:improved_crawler_detection)+ end++ def self.likely_crawler_condition(table: nil)+ prefix = table ? "#{table}." : ""+ "#{prefix}score > #{BOT_SCORE_THRESHOLD}"+ end+def self.score!(window_start:, window_end:)crawler_asns = SiteSetting.crawler_asns_map.map(&:to_i)crawler_detection_datacenter_asns =diff --git a/lib/dashboard_rollup_rebuilder.rb b/lib/dashboard_rollup_rebuilder.rbnew file mode 100644index 00000000000..1221568871b--- /dev/null+++ b/lib/dashboard_rollup_rebuilder.rb@@ -0,0 +1,90 @@+# frozen_string_literal: true++# Recomputes the rollup tables behind the admin dashboard from their source+# records. Replaying never drops data that can no longer be derived, because+# browser pageview events are pruned long before the rollups are. The flip side+# is that rows whose source events are gone are refreshed in place rather than+# removed, so a replay cannot retract counts that were once written.+class DashboardRollupRebuilder+ NAMES = %w[+ browser_pageview_country+ browser_pageview_referrer+ browser_pageview_session_engagement+ browser_pageview_crawler+ category_activity+ user_visit+ ].freeze++ def self.names+ NAMES+ end++ def self.known?(name)+ NAMES.include?(name.to_s)+ end++ def self.rebuild!(name = nil, io: $stdout)+ new(io: io).rebuild!(name)+ end++ def initialize(io: $stdout)+ @io = io+ end++ def rebuild!(name = nil)+ rollups = name ? [name.to_s] : NAMES++ rollups.each do |rollup|+ log "Rebuilding #{rollup} rollups for #{RailsMultisite::ConnectionManagement.current_db}"+ public_send(:"rebuild_#{rollup}")+ end+ end++ def rebuild_browser_pageview_country+ aggregate_browser_pageviews(BrowserPageviewCountryDailyRollup)+ end++ def rebuild_browser_pageview_referrer+ aggregate_browser_pageviews(BrowserPageviewReferrerDailyRollup)+ end++ def rebuild_browser_pageview_session_engagement+ aggregate_browser_pageviews(BrowserPageviewSessionEngagementDailyRollup)+ end++ def rebuild_browser_pageview_crawler+ aggregate_browser_pageviews(BrowserPageviewCrawlerDailyRollup)+ end++ def rebuild_category_activity+ CategoryActivityDailyRollup.rebuild!+ end++ def rebuild_user_visit+ start_date = UserVisit.minimum(:visited_at)&.to_date+ return log(" no user visits, skipping") if start_date.nil?++ UserVisitDailyRollup.aggregate(start_date: start_date, end_date: Time.zone.today)+ end++ private++ def aggregate_browser_pageviews(rollup)+ start_date = earliest_event_date+ return log(" no browser pageview events, skipping") if start_date.nil?++ rollup.aggregate(start_date: start_date, end_date: Time.zone.today)+ end++ def earliest_event_date+ @earliest_event_date ||=+ BrowserPageviewEvent+ .where(BrowserPageviewEvent.rollup_source_condition)+ .minimum(:created_at)+ &.to_date+ end++ def log(message)+ @io.puts(message)+ end+enddiff --git a/lib/search.rb b/lib/search.rbindex 963e3e1a0d1..f18116abdb0 100644--- a/lib/search.rb+++ b/lib/search.rb@@ -336,6 +336,7 @@ class Searchip_address: @opts[:ip_address],user_agent: @opts[:user_agent],user_id: @opts[:user_id],+ session_id: @opts[:session_id],)@results.search_log_id = search_log_id unless status == :errorenddiff --git a/lib/tasks/admin_dashboard.rake b/lib/tasks/admin_dashboard.rakenew file mode 100644index 00000000000..e8403822ebb--- /dev/null+++ b/lib/tasks/admin_dashboard.rake@@ -0,0 +1,38 @@+# frozen_string_literal: true++def each_dashboard_db(&block)+ if ENV["RAILS_DB"]+ block.call+ else+ RailsMultisite::ConnectionManagement.each_connection(&block)+ end+end++desc "Rebuild admin dashboard rollup tables. Pass a name to rebuild just one."+task "admin_dashboard:rebuild_rollups", [:name] => :environment do |_, args|+ name = args[:name].presence++ if name && !DashboardRollupRebuilder.known?(name)+ abort "Unknown rollup '#{name}'. Known rollups: #{DashboardRollupRebuilder.names.join(", ")}"+ end++ each_dashboard_db { DashboardRollupRebuilder.rebuild!(name) }+end++desc "Classify existing search logs from known crawler user agents as crawler traffic"+task "admin_dashboard:backfill_search_log_crawlers" => :environment do+ each_dashboard_db do+ db = RailsMultisite::ConnectionManagement.current_db+ flagged = SearchLog.backfill_crawler!+ puts "#{db}: flagged #{flagged} search logs from known crawler user agents"++ next if !CrawlerScorer.enabled?++ likely =+ SearchLog.flag_likely_crawlers!(+ window_start: BrowserPageviewEvent.retention_cutoff,+ window_end: Time.zone.now,+ )+ puts "#{db}: flagged #{likely} more from sessions scored as likely crawlers"+ end+enddiff --git a/spec/jobs/detect_crawler_pageviews_spec.rb b/spec/jobs/detect_crawler_pageviews_spec.rbindex 40abf4e3401..a351c865c13 100644--- a/spec/jobs/detect_crawler_pageviews_spec.rb+++ b/spec/jobs/detect_crawler_pageviews_spec.rb@@ -16,4 +16,20 @@ RSpec.describe Jobs::DetectCrawlerPageviews dodescribed_class.new.execute({})end++ it "flags searches made from sessions the scoring pass marked as likely crawlers" do+ SiteSetting.improved_crawler_detection = true+ session_id = SecureRandom.hex(16)+ Fabricate(+ :browser_pageview_event,+ session_id: session_id,+ user_agent: "Mozilla/5.0 (X11; Linux x86_64) HeadlessChrome/120.0.0.0",+ created_at: 45.minutes.ago,+ )+ log = Fabricate(:search_log, user: nil, session_id: session_id, created_at: 45.minutes.ago)++ described_class.new.execute({})++ expect(log.reload.likely_crawler).to eq(true)+ endenddiff --git a/spec/lib/dashboard_rollup_rebuilder_spec.rb b/spec/lib/dashboard_rollup_rebuilder_spec.rbnew file mode 100644index 00000000000..9738dda5f23--- /dev/null+++ b/spec/lib/dashboard_rollup_rebuilder_spec.rb@@ -0,0 +1,59 @@+# frozen_string_literal: true++RSpec.describe DashboardRollupRebuilder do+ subject(:rebuilder) { described_class.new(io: StringIO.new) }++ describe "#rebuild!" do+ it "rebuilds country and referrer rollups from the events they summarise" do+ Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")+ BrowserPageviewCountryDailyRollup.aggregate(start_date: 1.day.ago, end_date: Time.zone.today)+ BrowserPageviewReferrerDailyRollup.aggregate(start_date: 1.day.ago, end_date: Time.zone.today)+ BrowserPageviewCountryDailyRollup.delete_all+ BrowserPageviewReferrerDailyRollup.delete_all++ rebuilder.rebuild!++ expect(BrowserPageviewCountryDailyRollup.sum(:count)).to eq(1)+ expect(BrowserPageviewReferrerDailyRollup.sum(:count)).to eq(1)+ end++ it "populates likely crawler counts on rollups that were built without them" do+ SiteSetting.improved_crawler_detection = true+ Fabricate(:browser_pageview_event, country_code: "US", score: 90)+ BrowserPageviewCountryDailyRollup.aggregate(start_date: 1.day.ago, end_date: Time.zone.today)+ BrowserPageviewCountryDailyRollup.update_all(likely_crawler_count: 0)++ rebuilder.rebuild!("browser_pageview_country")++ expect(BrowserPageviewCountryDailyRollup.sum(:likely_crawler_count)).to eq(1)+ end++ it "rebuilds only the named rollup" do+ Fabricate(:browser_pageview_event, country_code: "US")++ rebuilder.rebuild!("browser_pageview_country")++ expect(BrowserPageviewCountryDailyRollup.count).to eq(1)+ expect(BrowserPageviewReferrerDailyRollup.count).to eq(0)+ end++ it "leaves browser pageview rollups empty when no events exist" do+ expect { rebuilder.rebuild!("browser_pageview_country") }.not_to change {+ BrowserPageviewCountryDailyRollup.count+ }+ end++ it "leaves user visit rollups empty when nobody has visited" do+ UserVisit.delete_all++ expect { rebuilder.rebuild!("user_visit") }.not_to change { UserVisitDailyRollup.count }+ end+ end++ describe ".known?" do+ it "recognises every rollup it can rebuild and rejects anything else" do+ expect(described_class.names).to all(satisfy { |name| described_class.known?(name) })+ expect(described_class.known?("nope")).to eq(false)+ end+ end+enddiff --git a/spec/models/browser_pageview_country_daily_rollup_spec.rb b/spec/models/browser_pageview_country_daily_rollup_spec.rbindex 0564a6dde1c..65196c35e14 100644--- a/spec/models/browser_pageview_country_daily_rollup_spec.rb+++ b/spec/models/browser_pageview_country_daily_rollup_spec.rb@@ -22,17 +22,21 @@ RSpec.describe BrowserPageviewCountryDailyRollup do)end- it "splits total count from logged-in count" do+ it "splits the total count by logged-in and likely crawler traffic" douser = Fabricate(:user)- Fabricate(:browser_pageview_event, country_code: "US", user_id: user.id)- Fabricate(:browser_pageview_event, country_code: "US")- Fabricate(:browser_pageview_event, country_code: "US")+ Fabricate(:browser_pageview_event, country_code: "US", score: 90)+ Fabricate(:browser_pageview_event, country_code: "US", score: 90, user_id: user.id)+ Fabricate(:browser_pageview_event, country_code: "US", score: 10)+ Fabricate(:browser_pageview_event, country_code: "US", score: nil, user_id: user.id)described_class.aggregate(start_date: start_date, end_date: end_date)- row = described_class.first- expect(row.count).to eq(3)- expect(row.logged_in_count).to eq(1)+ expect(described_class.find_by(country_code: "US")).to have_attributes(+ count: 4,+ logged_in_count: 2,+ likely_crawler_count: 2,+ likely_crawler_logged_in_count: 1,+ )endit "stores rows for NULL country_code (no GeoIP match) without duplication on re-run" dodiff --git a/spec/models/browser_pageview_referrer_daily_rollup_spec.rb b/spec/models/browser_pageview_referrer_daily_rollup_spec.rbindex c340cc65da2..3ba40b3a51a 100644--- a/spec/models/browser_pageview_referrer_daily_rollup_spec.rb+++ b/spec/models/browser_pageview_referrer_daily_rollup_spec.rb@@ -16,16 +16,25 @@ RSpec.describe BrowserPageviewReferrerDailyRollup doexpect(rollups).to contain_exactly(["google.com", 2], ["reddit.com", 1])end- it "splits total count from logged-in count" do+ it "splits the total count by logged-in and likely crawler traffic" douser = Fabricate(:user)- Fabricate(:browser_pageview_event, normalized_referrer: "google.com", user_id: user.id)- Fabricate(:browser_pageview_event, normalized_referrer: "google.com")+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com", score: 90)+ Fabricate(+ :browser_pageview_event,+ normalized_referrer: "google.com",+ score: 90,+ user_id: user.id,+ )+ Fabricate(:browser_pageview_event, normalized_referrer: "google.com", score: 10)described_class.aggregate(start_date: start_date, end_date: end_date)- row = described_class.find_by(normalized_referrer: "google.com")- expect(row.count).to eq(2)- expect(row.logged_in_count).to eq(1)+ expect(described_class.find_by(normalized_referrer: "google.com")).to have_attributes(+ count: 3,+ logged_in_count: 1,+ likely_crawler_count: 2,+ likely_crawler_logged_in_count: 1,+ )endit "stores NULL-referrer rows (direct visits)" dodiff --git a/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb b/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rbindex 60dce237463..3b7572c26e5 100644--- a/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb+++ b/spec/models/browser_pageview_session_engagement_daily_rollup_spec.rb@@ -285,5 +285,68 @@ RSpec.describe BrowserPageviewSessionEngagementDailyRollup dohave_attributes(sessions: 2, bounced: 2, engaged_seconds_total: 10),)end++ it "records the likely crawler share of sessions, bounces, and engaged seconds" do+ crawler_event =+ Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 10, 9), score: 90)+ Fabricate(+ :browser_pageview_session_engagement,+ session_id: crawler_event.session_id,+ engaged_seconds: 3,+ )++ human_event =+ Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 11, 10), score: 10)+ Fabricate(+ :browser_pageview_session_engagement,+ session_id: human_event.session_id,+ engaged_seconds: 40,+ )++ described_class.aggregate(start_date:, end_date:)++ expect(described_class.all).to contain_exactly(+ have_attributes(+ date: Date.new(2026, 6, 10),+ sessions: 1,+ bounced: 1,+ engaged_seconds_total: 3,+ likely_crawler_sessions: 1,+ likely_crawler_bounced: 1,+ likely_crawler_engaged_seconds_total: 3,+ ),+ have_attributes(+ date: Date.new(2026, 6, 11),+ sessions: 1,+ bounced: 0,+ engaged_seconds_total: 40,+ likely_crawler_sessions: 0,+ likely_crawler_bounced: 0,+ likely_crawler_engaged_seconds_total: 0,+ ),+ )+ end++ it "treats a session as a likely crawler when any of its pageviews scores above the threshold" do+ session_id = SecureRandom.hex(16)+ Fabricate(+ :browser_pageview_event,+ session_id: session_id,+ created_at: Time.utc(2026, 6, 10, 9),+ score: 10,+ )+ Fabricate(+ :browser_pageview_event,+ session_id: session_id,+ created_at: Time.utc(2026, 6, 10, 10),+ score: 90,+ )++ described_class.aggregate(start_date:, end_date:)++ expect(described_class.all).to contain_exactly(+ have_attributes(sessions: 1, likely_crawler_sessions: 1),+ )+ endendenddiff --git a/spec/models/category_activity_daily_rollup_spec.rb b/spec/models/category_activity_daily_rollup_spec.rbindex 555aaeea144..b6e0beab1b7 100644--- a/spec/models/category_activity_daily_rollup_spec.rb+++ b/spec/models/category_activity_daily_rollup_spec.rb@@ -25,6 +25,111 @@ RSpec.describe CategoryActivityDailyRollup do)end+ it "records likely crawler pageviews from browser events separately" do+ topic = Fabricate(:topic, category: category, created_at: 3.days.ago)+ TopicViewStat.create!(+ topic: topic,+ viewed_at: 3.days.ago.to_date,+ anonymous_views: 7,+ logged_in_views: 3,+ )+ Fabricate(+ :browser_pageview_event,+ topic_id: topic.id,+ ip_address: "1.1.1.1",+ created_at: 3.days.ago,+ score: 90,+ )+ Fabricate(+ :browser_pageview_event,+ topic_id: topic.id,+ ip_address: "2.2.2.2",+ created_at: 3.days.ago,+ score: 90,+ )+ Fabricate(:browser_pageview_event, topic_id: topic.id, created_at: 3.days.ago, score: 10)++ described_class.aggregate(start_date: 5.days.ago, end_date: Time.zone.today)++ expect(described_class.find_by(category: category, date: 3.days.ago.to_date)).to(+ have_attributes(page_views: 10, likely_crawler_page_views: 2),+ )+ end++ it "counts a likely crawler revisiting the same topic once a day, matching topic view stats" do+ topic = Fabricate(:topic, category: category, created_at: 3.days.ago)+ TopicViewStat.create!(+ topic: topic,+ viewed_at: 3.days.ago.to_date,+ anonymous_views: 5,+ logged_in_views: 0,+ )+ 2.times do+ Fabricate(+ :browser_pageview_event,+ topic_id: topic.id,+ ip_address: "1.1.1.1",+ created_at: 3.days.ago,+ score: 90,+ )+ end++ described_class.aggregate(start_date: 5.days.ago, end_date: Time.zone.today)++ expect(described_class.find_by(category: category, date: 3.days.ago.to_date)).to(+ have_attributes(page_views: 5, likely_crawler_page_views: 1),+ )+ end++ it "does not create a row for a day whose only signal is likely crawler pageviews" do+ topic = Fabricate(:topic, category: category, created_at: 10.days.ago)+ Fabricate(:browser_pageview_event, topic_id: topic.id, created_at: 3.days.ago, score: 90)++ described_class.aggregate(start_date: 5.days.ago, end_date: Time.zone.today)++ expect(described_class.where(date: 3.days.ago.to_date)).to be_empty+ end++ it "keeps likely crawler pageviews for dates whose source events have been pruned" do+ topic = Fabricate(:topic, category: category, created_at: 3.days.ago)+ TopicViewStat.create!(+ topic: topic,+ viewed_at: 3.days.ago.to_date,+ anonymous_views: 10,+ logged_in_views: 0,+ )+ Fabricate(:browser_pageview_event, topic_id: topic.id, created_at: 3.days.ago, score: 90)+ described_class.aggregate(start_date: 5.days.ago, end_date: Time.zone.today)++ BrowserPageviewEvent.delete_all+ described_class.aggregate(start_date: 5.days.ago, end_date: Time.zone.today)++ expect(described_class.find_by(category: category, date: 3.days.ago.to_date)).to(+ have_attributes(page_views: 10, likely_crawler_page_views: 1),+ )+ end++ it "recomputes likely crawler pageviews downward while the source events remain" do+ topic = Fabricate(:topic, category: category, created_at: 3.days.ago)+ TopicViewStat.create!(+ topic: topic,+ viewed_at: 3.days.ago.to_date,+ anonymous_views: 10,+ logged_in_views: 0,+ )+ crawler_event =+ Fabricate(:browser_pageview_event, topic_id: topic.id, created_at: 3.days.ago, score: 90)+ Fabricate(:browser_pageview_event, topic_id: topic.id, created_at: 3.days.ago, score: 10)+ described_class.aggregate(start_date: 5.days.ago, end_date: Time.zone.today)++ crawler_event.update!(score: 10)+ described_class.aggregate(start_date: 5.days.ago, end_date: Time.zone.today)++ expect(described_class.find_by(category: category, date: 3.days.ago.to_date)).to(+ have_attributes(page_views: 10, likely_crawler_page_views: 0),+ )+ end+it "ignores deleted topics, deleted posts and private messages" doFabricate(:topic, category: category, created_at: 1.day.ago, deleted_at: Time.zone.now)Fabricate(:private_message_topic, category: category, created_at: 1.day.ago)@@ -139,6 +244,72 @@ RSpec.describe CategoryActivityDailyRollup doexpect(row.topics_current).to be_a(Integer)end+ it "subtracts likely crawler pageviews once crawler detection is enabled" do+ SiteSetting.improved_crawler_detection = true+ Fabricate(+ :category_activity_daily_rollup,+ category: category,+ date: 2.days.ago,+ topics: 0,+ posts: 0,+ page_views: 10,+ likely_crawler_page_views: 4,+ )++ row =+ described_class.period_totals(+ prev_start: 14.days.ago.to_date,+ current_start: 7.days.ago.to_date,+ current_end: Time.zone.today,+ ).first++ expect(row.page_views_current).to eq(6)+ end++ it "clamps to zero when likely crawler pageviews overshoot the topic view stats" do+ SiteSetting.improved_crawler_detection = true+ Fabricate(+ :category_activity_daily_rollup,+ category: category,+ date: 2.days.ago,+ topics: 1,+ posts: 0,+ page_views: 3,+ likely_crawler_page_views: 9,+ )++ row =+ described_class.period_totals(+ prev_start: 14.days.ago.to_date,+ current_start: 7.days.ago.to_date,+ current_end: Time.zone.today,+ ).first++ expect(row.page_views_current).to eq(0)+ end++ it "counts likely crawler pageviews while crawler detection is disabled" do+ SiteSetting.improved_crawler_detection = false+ Fabricate(+ :category_activity_daily_rollup,+ category: category,+ date: 2.days.ago,+ topics: 0,+ posts: 0,+ page_views: 10,+ likely_crawler_page_views: 4,+ )++ row =+ described_class.period_totals(+ prev_start: 14.days.ago.to_date,+ current_start: 7.days.ago.to_date,+ current_end: Time.zone.today,+ ).first++ expect(row.page_views_current).to eq(10)+ end+it "excludes categories outside the requested ids" doother_category = Fabricate(:category)Fabricate(:category_activity_daily_rollup, category: category)diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rbindex c2fbad34dc8..4483b95b040 100644--- a/spec/models/report_spec.rb+++ b/spec/models/report_spec.rb@@ -1974,6 +1974,19 @@ RSpec.describe Report doendend+ describe ".cache_key" do+ it "includes the crawler detection state in the cache key" do+ report = Report.find("signups")++ SiteSetting.improved_crawler_detection = false+ disabled_key = Report.cache_key(report)++ SiteSetting.improved_crawler_detection = true++ expect(Report.cache_key(report)).not_to eq(disabled_key)+ end+ end+describe "top_uploads" docontext "with no data" doit "works" dodiff --git a/spec/models/reports/top_countries_by_browser_pageviews_spec.rb b/spec/models/reports/top_countries_by_browser_pageviews_spec.rbindex 61131dc1f31..f345652d5aa 100644--- a/spec/models/reports/top_countries_by_browser_pageviews_spec.rb+++ b/spec/models/reports/top_countries_by_browser_pageviews_spec.rb@@ -13,7 +13,7 @@ describe Reports::TopCountriesByBrowserPageviews doit "ranks countries by event count and computes percent of total browser pageviews" do3.times { Fabricate(:browser_pageview_event, country_code: "US") }- 1.times { Fabricate(:browser_pageview_event, country_code: "GB") }+ Fabricate(:browser_pageview_event, country_code: "GB")data = report.dataexpect(data.map { |row| row[:country_code] }).to eq(%w[US GB])@@ -52,6 +52,25 @@ describe Reports::TopCountriesByBrowserPageviews doexpect(data.first[:percent]).to eq(50)end+ it "excludes likely crawler pageviews once crawler detection is enabled" do+ SiteSetting.improved_crawler_detection = true+ 3.times { Fabricate(:browser_pageview_event, country_code: "US", score: 10) }+ Fabricate(:browser_pageview_event, country_code: "GB", score: 90)+ Fabricate(:browser_pageview_event, country_code: "GB", score: 10)++ data = report.data+ expect(data.map { |row| row[:country_code] }).to eq(%w[US GB])+ expect(data.map { |row| row[:count] }).to eq([3, 1])+ expect(data.first[:percent]).to eq(75)+ end++ it "counts likely crawler pageviews while crawler detection is disabled" do+ SiteSetting.improved_crawler_detection = false+ 3.times { Fabricate(:browser_pageview_event, country_code: "US", score: 90) }++ expect(report.data.first[:count]).to eq(3)+ end+it "returns empty data when no events exist in range" doexpect(report.data).to eq([])enddiff --git a/spec/models/reports/top_referrers_by_browser_pageviews_spec.rb b/spec/models/reports/top_referrers_by_browser_pageviews_spec.rbindex 1e19d459e13..42d2bd4693a 100644--- a/spec/models/reports/top_referrers_by_browser_pageviews_spec.rb+++ b/spec/models/reports/top_referrers_by_browser_pageviews_spec.rb@@ -17,7 +17,7 @@ describe Reports::TopReferrersByBrowserPageviews do3.times doFabricate(:browser_pageview_event, normalized_referrer: "news.ycombinator.com/item?id=1")end- 1.times { Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com/r/discourse") }+ Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com/r/discourse")data = report.dataexpect(data.map { |row| row[:normalized_referrer] }).to eq(@@ -91,6 +91,25 @@ describe Reports::TopReferrersByBrowserPageviews doexpect(report.data.first[:count]).to eq(1)end+ it "excludes likely crawler pageviews once crawler detection is enabled" do+ SiteSetting.improved_crawler_detection = true+ 3.times { Fabricate(:browser_pageview_event, normalized_referrer: "google.com", score: 10) }+ Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com", score: 90)+ Fabricate(:browser_pageview_event, normalized_referrer: "reddit.com", score: 10)++ data = report.data+ expect(data.map { |row| row[:normalized_referrer] }).to eq(%w[google.com reddit.com])+ expect(data.map { |row| row[:count] }).to eq([3, 1])+ expect(data.first[:percent]).to eq(75)+ end++ it "counts likely crawler pageviews while crawler detection is disabled" do+ SiteSetting.improved_crawler_detection = false+ 3.times { Fabricate(:browser_pageview_event, normalized_referrer: "google.com", score: 90) }++ expect(report.data.first[:count]).to eq(3)+ end+it "returns empty data when no qualifying events exist" doFabricate(:browser_pageview_event, normalized_referrer: nil)expect(report.data).to eq([])diff --git a/spec/models/search_log_spec.rb b/spec/models/search_log_spec.rbindex 7f147e75041..35d965333af 100644--- a/spec/models/search_log_spec.rb+++ b/spec/models/search_log_spec.rb@@ -187,6 +187,182 @@ RSpec.describe SearchLog, type: :model doendend+ describe ".log" do+ it "flags an anonymous search from a known crawler user agent" do+ _status, id =+ SearchLog.log(+ term: "ruby",+ search_type: :header,+ ip_address: "127.0.0.1",+ user_agent: "Googlebot/2.1 (+http://www.google.com/bot.html)",+ )++ expect(SearchLog.find(id).crawler).to eq(true)+ end++ it "flags a logged-in search from a crawler user agent" do+ user = Fabricate(:user)+ _status, id =+ SearchLog.log(+ term: "ruby",+ search_type: :header,+ ip_address: "127.0.0.1",+ user_agent: "Googlebot/2.1 (+http://www.google.com/bot.html)",+ user_id: user.id,+ )++ expect(SearchLog.find(id).crawler).to eq(true)+ end++ it "flags a search from a scripted client that is not a browser" do+ _status, id =+ SearchLog.log(+ term: "ruby",+ search_type: :header,+ ip_address: "127.0.0.1",+ user_agent: "python-requests/2.31.0",+ )++ expect(SearchLog.find(id).crawler).to eq(true)+ end++ it "records the pageview session the search was made from" do+ _status, id =+ SearchLog.log(+ term: "ruby",+ search_type: :header,+ ip_address: "127.0.0.1",+ session_id: "a" * (SearchLog::MAXIMUM_SESSION_ID_LENGTH + 10),+ )++ expect(SearchLog.find(id).session_id).to eq("a" * SearchLog::MAXIMUM_SESSION_ID_LENGTH)+ end++ it "does not flag an anonymous search that arrived without a user agent" do+ _status, id = SearchLog.log(term: "ruby", search_type: :header, ip_address: "127.0.0.1")++ expect(SearchLog.find(id).crawler).to eq(false)+ end++ it "does not flag an anonymous search from a browser user agent" do+ _status, id =+ SearchLog.log(+ term: "ruby",+ search_type: :header,+ ip_address: "127.0.0.1",+ user_agent:+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",+ )++ expect(SearchLog.find(id).crawler).to eq(false)+ end+ end++ describe ".flag_likely_crawlers!" do+ let(:session_id) { SecureRandom.hex(16) }+ let(:crawler_score) { CrawlerScorer::BOT_SCORE_THRESHOLD + 1 }++ def flag!+ described_class.flag_likely_crawlers!(window_start: 1.hour.ago, window_end: Time.now)+ end++ it "flags a search made from a session scored as a likely crawler" do+ Fabricate(:browser_pageview_event, session_id: session_id, score: crawler_score)+ log = Fabricate(:search_log, user: nil, session_id: session_id, created_at: 30.minutes.ago)++ flag!++ expect(log.reload.likely_crawler).to eq(true)+ end++ it "leaves a search whose session was scored below the crawler threshold" do+ Fabricate(+ :browser_pageview_event,+ session_id: session_id,+ score: CrawlerScorer::BOT_SCORE_THRESHOLD,+ )+ log = Fabricate(:search_log, user: nil, session_id: session_id, created_at: 30.minutes.ago)++ flag!++ expect(log.reload.likely_crawler).to eq(false)+ end++ it "leaves a search that carries no session" do+ Fabricate(:browser_pageview_event, session_id: session_id, score: crawler_score)+ log = Fabricate(:search_log, user: nil, session_id: nil, created_at: 30.minutes.ago)++ flag!++ expect(log.reload.likely_crawler).to eq(false)+ end++ it "leaves a search whose session has no scored pageviews at all" do+ Fabricate(:browser_pageview_event, session_id: SecureRandom.hex(16), score: crawler_score)+ log = Fabricate(:search_log, user: nil, session_id: session_id, created_at: 30.minutes.ago)++ flag!++ expect(log.reload.likely_crawler).to eq(false)+ end++ it "flags a logged-in search whose session is scored as a likely crawler" do+ Fabricate(:browser_pageview_event, session_id: session_id, score: crawler_score)+ log =+ Fabricate(+ :search_log,+ user: Fabricate(:user),+ ip_address: nil,+ session_id: session_id,+ created_at: 30.minutes.ago,+ )++ flag!++ expect(log.reload.likely_crawler).to eq(true)+ end++ it "leaves searches outside the window alone" do+ Fabricate(:browser_pageview_event, session_id: session_id, score: crawler_score)+ log = Fabricate(:search_log, user: nil, session_id: session_id, created_at: 5.hours.ago)++ flag!++ expect(log.reload.likely_crawler).to eq(false)+ end+ end++ describe ".backfill_crawler!" do+ it "flags existing searches from known crawler user agents" do+ crawler =+ Fabricate(+ :search_log,+ user: nil,+ user_agent: "Googlebot/2.1 (+http://www.google.com/bot.html)",+ )+ human = Fabricate(:search_log, user: nil, user_agent: "Mozilla/5.0 (Macintosh) Chrome/120.0")+ no_agent = Fabricate(:search_log, user: nil, user_agent: nil)+ blank_agent = Fabricate(:search_log, user: nil, user_agent: "")+ member = Fabricate(:search_log, user: Fabricate(:user), user_agent: "Googlebot/2.1")++ expect(described_class.backfill_crawler!).to eq(2)++ expect(crawler.reload.crawler).to eq(true)+ expect(human.reload.crawler).to eq(false)+ expect(no_agent.reload.crawler).to eq(false)+ expect(blank_agent.reload.crawler).to eq(false)+ expect(member.reload.crawler).to eq(true)+ end++ it "is idempotent" do+ Fabricate(:search_log, user: nil, user_agent: "Googlebot/2.1")++ described_class.backfill_crawler!++ expect(described_class.backfill_crawler!).to eq(0)+ end+ end+describe ".term_details" doit "should only use the date for the period" dotime = Time.utc(2019, 5, 23, 18, 15, 30)@@ -241,17 +417,42 @@ RSpec.describe SearchLog, type: :model doSearchLog.term_details("ruby", :weekly, :non_staff_only)[:data].sum { |point| point[:y] },).to eq(1)end++ it "returns non-staff and anonymous searches minus crawlers with the human_only search type" do+ SiteSetting.improved_crawler_detection = true+ member = Fabricate(:user)+ Fabricate(:search_log, term: "ruby", user: member)+ Fabricate(:search_log, term: "ruby", user: Fabricate(:admin))+ Fabricate(:search_log, term: "ruby", user: Fabricate(:moderator))+ Fabricate(:search_log, term: "ruby", user: nil)+ Fabricate(:search_log, term: "ruby", user: nil, crawler: true)++ expect(+ SearchLog.term_details("ruby", :weekly, :human_only)[:data].sum { |point| point[:y] },+ ).to eq(2)+ endenddescribe "trending" dofab!(:user)+ def log_search(term, **opts)+ SearchLog.log(+ term: term,+ search_type: :header,+ ip_address: "127.0.0.1",+ user_agent:+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",+ **opts,+ )+ end+before do- SearchLog.log(term: "ruby", search_type: :header, ip_address: "127.0.0.1")- SearchLog.log(term: "php", search_type: :header, ip_address: "127.0.0.1")- SearchLog.log(term: "java", search_type: :header, ip_address: "127.0.0.1")- SearchLog.log(term: "ruby", search_type: :header, ip_address: "127.0.0.1", user_id: user.id)- SearchLog.log(term: "swift", search_type: :header, ip_address: "127.0.0.1")- SearchLog.log(term: "ruby", search_type: :header, ip_address: "127.0.0.2")+ log_search("ruby")+ log_search("php")+ log_search("java")+ log_search("ruby", user_id: user.id)+ log_search("swift")+ log_search("ruby", ip_address: "127.0.0.2")endit "considers time period" do@@ -285,6 +486,28 @@ RSpec.describe SearchLog, type: :model doexpect(results.map { |trend| [trend.term, trend.searches] }).to eq([["ruby", 1]])end++ it "returns non-staff and anonymous searches minus crawlers with the human_only search type" do+ SiteSetting.improved_crawler_detection = true+ Fabricate(:search_log, term: "admin-search", user: Fabricate(:admin))+ Fabricate(:search_log, term: "crawler-search", user: nil, crawler: true)++ results = SearchLog.trending(:all, :human_only).to_a++ expect(results.map { |trend| [trend.term, trend.searches] }).to eq(+ [["ruby", 3], ["java", 1], ["php", 1], ["swift", 1]],+ )+ end++ it "falls back to members-only for human_only while crawler detection is disabled" do+ SiteSetting.improved_crawler_detection = false+ Fabricate(:search_log, term: "admin-search", user: Fabricate(:admin))+ Fabricate(:search_log, term: "anonymous-search", user: nil)++ results = SearchLog.trending(:all, :human_only).to_a++ expect(results.map { |trend| [trend.term, trend.searches] }).to eq([["ruby", 1]])+ endenddescribe "clean_up" dodiff --git a/spec/requests/admin/dashboard_controller_spec.rb b/spec/requests/admin/dashboard_controller_spec.rbindex 423d8f20be3..7a3054d07ff 100644--- a/spec/requests/admin/dashboard_controller_spec.rb+++ b/spec/requests/admin/dashboard_controller_spec.rb@@ -336,6 +336,7 @@ RSpec.describe Admin::DashboardController doexpect(response.status).to eq(200)expect(search_data).to eq("logging_enabled" => true,+ "search_type" => "non_staff_only","kpis" => {"total_searches" => {"value" => 2,diff --git a/spec/requests/admin/search_logs_spec.rb b/spec/requests/admin/search_logs_spec.rbindex 1946eaf5efe..581a4fc0aba 100644--- a/spec/requests/admin/search_logs_spec.rb+++ b/spec/requests/admin/search_logs_spec.rb@@ -50,6 +50,20 @@ RSpec.describe Admin::SearchLogsController do[["member-search", 1]],)end++ it "returns non-staff and anonymous searches minus crawlers with the human_only search type" do+ SiteSetting.improved_crawler_detection = true+ Fabricate(:search_log, term: "member-search", user: user)+ Fabricate(:search_log, term: "admin-search", user: admin)+ Fabricate(:search_log, term: "anonymous-search", user: nil)+ Fabricate(:search_log, term: "crawler-search", user: nil, crawler: true)++ get "/admin/logs/search_logs.json", params: { search_type: "human_only" }++ expect(response.parsed_body.map { |entry| [entry["term"], entry["searches"]] }).to eq(+ [["anonymous-search", 1], ["member-search", 1], ["ruby", 1]],+ )+ endendcontext "when logged in as a moderator" dodiff --git a/spec/requests/search_controller_spec.rb b/spec/requests/search_controller_spec.rbindex bd2b4599e32..2a51527944b 100644--- a/spec/requests/search_controller_spec.rb+++ b/spec/requests/search_controller_spec.rb@@ -275,6 +275,22 @@ RSpec.describe SearchController doexpect(log.term).to eq("wookie")end+ it "logs the pageview session the search was made from" do+ SiteSetting.log_search_queries = true+ session_id = SecureRandom.alphanumeric(32)++ get "/search/query.json",+ params: {+ term: "wookie",+ },+ headers: {+ "Discourse-Pageview-Session-Id" => session_id,+ }++ expect(response.status).to eq(200)+ expect(SearchLog.find_by(term: "wookie").session_id).to eq(session_id)+ end+it "doesn't log when disabled" doSiteSetting.log_search_queries = falseget "/search/query.json", params: { term: "wookie" }diff --git a/spec/services/admin_dashboard_engagement_spec.rb b/spec/services/admin_dashboard_engagement_spec.rbindex 54c520f6edc..db06c62c410 100644--- a/spec/services/admin_dashboard_engagement_spec.rb+++ b/spec/services/admin_dashboard_engagement_spec.rb@@ -237,6 +237,12 @@ describe AdminDashboardEngagement doenddescribe "activity_by_category" do+ def page_views_for(result, category)+ result[:activity_by_category][:rows].find { |row| row[:category_id] == category.id }[+ :page_views+ ]+ end+it "includes the activity_by_category block with rows and total" doresult = described_class.build(start_date: "2026-04-01", end_date: "2026-04-28")activity = result[:activity_by_category]@@ -245,6 +251,27 @@ describe AdminDashboardEngagement doexpect(activity).to have_key(:total)end+ it "invalidates the cached rows when crawler detection is toggled" do+ SiteSetting.improved_crawler_detection = false+ category = Fabricate(:category)+ Fabricate(+ :category_activity_daily_rollup,+ category: category,+ date: "2026-04-10",+ topics: 0,+ posts: 0,+ page_views: 10,+ likely_crawler_page_views: 4,+ )++ first = described_class.build(start_date: "2026-04-01", end_date: "2026-04-28")+ expect(page_views_for(first, category)).to eq(10)++ SiteSetting.improved_crawler_detection = true+ second = described_class.build(start_date: "2026-04-01", end_date: "2026-04-28")+ expect(page_views_for(second, category)).to eq(6)+ end+it "honours category visibility when current_user is a moderator" domoderator = Fabricate(:moderator)private_group = Fabricate(:group)diff --git a/spec/services/admin_dashboard_search_spec.rb b/spec/services/admin_dashboard_search_spec.rbindex bd41283cecf..cde7fdb9537 100644--- a/spec/services/admin_dashboard_search_spec.rb+++ b/spec/services/admin_dashboard_search_spec.rb@@ -3,10 +3,13 @@RSpec.describe AdminDashboardSearch dofab!(:user)- before { freeze_time(Time.zone.local(2026, 5, 14, 12, 0, 0)) }+ before do+ freeze_time(Time.zone.local(2026, 5, 14, 12, 0, 0))+ SiteSetting.improved_crawler_detection = true+ enddescribe ".build" do- it "returns KPIs, trending terms, and content gaps for the selected dates, counting only non-staff users" do+ it "returns KPIs, trending terms, and content gaps for the selected dates, counting non-staff and anonymous searches" doadmin = Fabricate(:admin)moderator = Fabricate(:moderator)@@ -55,13 +58,26 @@ RSpec.describe AdminDashboardSearch do)Fabricate.times(5, :search_log, term: "ghost", user: user, created_at: "2026-04-26 11:00")- # Anonymous searches (likely crawlers) must be excluded from every metric and from the+ Fabricate.times(8, :clicked_search_log, term: "onboarding", created_at: "2026-05-05 10:00")+ Fabricate.times(2, :search_log, term: "onboarding", created_at: "2026-05-05 11:00")++ # Searches flagged as crawler traffic must stay out of every metric and out of the# prior-window deltas. If counted, "crawler-bait" would top trending, inflate the- # no-result rate, and surface as a content gap; the anonymous "ruby" rows would change- # its count and the percent change.- Fabricate.times(30, :search_log, term: "crawler-bait", created_at: "2026-05-05 10:00")- Fabricate.times(10, :search_log, term: "ruby", created_at: "2026-05-02 09:00")- Fabricate.times(20, :search_log, term: "crawler-bait", created_at: "2026-04-26 09:00")+ # no-result rate, and surface as a content gap.+ Fabricate.times(+ 30,+ :search_log,+ term: "crawler-bait",+ crawler: true,+ created_at: "2026-05-05 12:00",+ )+ Fabricate.times(+ 20,+ :search_log,+ term: "crawler-bait",+ crawler: true,+ created_at: "2026-04-26 09:00",+ )Fabricate(:search_log, term: "admin-search", user: admin, created_at: "2026-05-05 12:00")Fabricate(:clicked_search_log, term: "ruby", user: moderator, created_at: "2026-05-05 13:00")@@ -73,20 +89,22 @@ RSpec.describe AdminDashboardSearch do)expect(described_class.build(start_date: "2026-05-01", end_date: "2026-05-07")).to eq(logging_enabled: true,+ search_type: "human_only",kpis: {total_searches: {- value: 19,+ value: 29,previous_value: 10,- percent_change: 90,+ percent_change: 190,},no_result_rate: {- value: 21,+ value: 14,previous_value: 50,- point_change: -29,+ point_change: -36,exceeds_threshold: true,},},trending: [+ { term: "onboarding", searches: 10 },{ term: "ruby", searches: 6 },{ term: "markdown tables", searches: 5 },{ term: "zeta", searches: 4 },@@ -100,6 +118,19 @@ RSpec.describe AdminDashboardSearch do)end+ it "stays members-only while crawler detection is disabled" do+ SiteSetting.improved_crawler_detection = false+ Fabricate.times(3, :search_log, term: "ruby", user: user, created_at: "2026-05-02 10:00")+ Fabricate.times(9, :search_log, term: "crawler-bait", created_at: "2026-05-02 11:00")+ Fabricate.times(4, :search_log, term: "onboarding", created_at: "2026-05-02 12:00")++ result = described_class.build(start_date: "2026-05-01", end_date: "2026-05-07")++ expect(result[:kpis][:total_searches][:value]).to eq(3)+ expect(result[:trending]).to eq([{ term: "ruby", searches: 3 }])+ expect(result[:search_type]).to eq("non_staff_only")+ end+it "buckets terms by exact CTR boundaries" doFabricate(:clicked_search_log, term: "tiny-ctr", user: user, created_at: "2026-05-02 10:00")Fabricate.times(@@ -156,6 +187,7 @@ RSpec.describe AdminDashboardSearch doexpect(described_class.build(start_date: "2026-05-01", end_date: "2026-05-07")).to eq(logging_enabled: true,+ search_type: "human_only",kpis: {total_searches: {value: 136,@@ -196,6 +228,7 @@ RSpec.describe AdminDashboardSearch doexpect(described_class.build(start_date: "2026-05-01", end_date: "2026-05-07")).to eq(logging_enabled: true,+ search_type: "human_only",kpis: {total_searches: {value: 11,@@ -243,6 +276,7 @@ RSpec.describe AdminDashboardSearch doexpect(described_class.build(start_date: "2026-05-01", end_date: "2026-05-07")).to eq(logging_enabled: true,+ search_type: "human_only",kpis: {total_searches: {value: 0,diff --git a/spec/services/admin_dashboard_site_traffic_spec.rb b/spec/services/admin_dashboard_site_traffic_spec.rbindex 90f5f2fe73c..dc5f1eecfb4 100644--- a/spec/services/admin_dashboard_site_traffic_spec.rb+++ b/spec/services/admin_dashboard_site_traffic_spec.rb@@ -728,6 +728,20 @@ RSpec.describe AdminDashboardSiteTraffic doexpect(second[:top_countries][:rows]).to be_emptyend+ it "invalidates the cached payload when crawler detection is toggled" do+ SiteSetting.improved_crawler_detection = false+ 3.times { Fabricate(:browser_pageview_event, country_code: "US", score: 90) }+ Fabricate(:browser_pageview_event, country_code: "US", score: 10)+ aggregate_rollups++ first = build_traffic(start_date: nil, end_date: nil)+ expect(first[:top_countries][:rows].first[:count]).to eq(4)++ SiteSetting.improved_crawler_detection = true+ second = build_traffic(start_date: nil, end_date: nil)+ expect(second[:top_countries][:rows].first[:count]).to eq(1)+ end+it "invalidates the cached payload when current_hostname changes" doDiscourse.stubs(:current_hostname).returns("forum-a.example.com")Fabricate(:browser_pageview_event, normalized_referrer: "forum-b.example.com/path")@@ -827,6 +841,39 @@ RSpec.describe AdminDashboardSiteTraffic do)end+ it "excludes likely crawler pageviews from the direct traffic share once crawler detection is enabled" do+ SiteSetting.improved_crawler_detection = true+ 3.times do+ Fabricate(+ :browser_pageview_event,+ normalized_referrer: nil,+ score: 10,+ created_at: "2026-05-01",+ )+ end+ 9.times do+ Fabricate(+ :browser_pageview_event,+ normalized_referrer: "google.com",+ score: 90,+ created_at: "2026-05-01",+ )+ end+ Fabricate(+ :browser_pageview_event,+ normalized_referrer: "google.com",+ score: 10,+ created_at: "2026-05-01",+ )+ aggregate_referrer_rollups++ expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-03")[:kpis]).to include(+ direct_traffic: {+ value: 75,+ },+ )+ end+it "reports zero direct traffic when every tracked pageview had a referrer" do4.times doFabricate(@@ -987,6 +1034,54 @@ RSpec.describe AdminDashboardSiteTraffic do)end+ it "excludes likely crawler sessions once crawler detection is enabled" do+ SiteSetting.improved_crawler_detection = true+ Fabricate(+ :browser_pageview_session_engagement_daily_rollup,+ date: Date.new(2026, 5, 10),+ logged_in: false,+ sessions: 14,+ bounced: 11,+ engaged_seconds_total: 700,+ likely_crawler_sessions: 6,+ likely_crawler_bounced: 6,+ likely_crawler_engaged_seconds_total: 100,+ )++ expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-14")[:kpis]).to include(+ bounce_rate: {+ value: 63,+ },+ average_session_duration_seconds: {+ value: 75,+ },+ )+ end++ it "counts likely crawler sessions while crawler detection is disabled" do+ SiteSetting.improved_crawler_detection = false+ Fabricate(+ :browser_pageview_session_engagement_daily_rollup,+ date: Date.new(2026, 5, 10),+ logged_in: false,+ sessions: 14,+ bounced: 11,+ engaged_seconds_total: 700,+ likely_crawler_sessions: 6,+ likely_crawler_bounced: 6,+ likely_crawler_engaged_seconds_total: 100,+ )++ expect(build_traffic(start_date: "2026-05-01", end_date: "2026-05-14")[:kpis]).to include(+ bounce_rate: {+ value: 79,+ },+ average_session_duration_seconds: {+ value: 50,+ },+ )+ end+it "reports zero average duration, not a placeholder, for visits with no engaged time" doFabricate(:browser_pageview_session_engagement_daily_rollup,@@ -1175,7 +1270,7 @@ RSpec.describe AdminDashboardSiteTraffic do)end- it "excludes anonymous crawlers when login is required" do+ it "excludes anonymous likely crawlers when login is required" doSiteSetting.login_required = trueFabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 10)diff --git a/spec/system/admin_dashboard_redesign/search_section_spec.rb b/spec/system/admin_dashboard_redesign/search_section_spec.rbindex f37b0b57970..902f09f39a6 100644--- a/spec/system/admin_dashboard_redesign/search_section_spec.rb+++ b/spec/system/admin_dashboard_redesign/search_section_spec.rb@@ -9,6 +9,7 @@ describe "Admin Dashboard Redesign | Search section" dobefore doSiteSetting.dashboard_improvements = true+ SiteSetting.improved_crawler_detection = trueAdminDashboardSectionConfiguration.update([{ id: "search", visible: true },@@ -22,51 +23,42 @@ describe "Admin Dashboard Redesign | Search section" dosign_in(current_user)end- it "lets staff review non-staff search health, inspect tooltips, and drill into terms",+ it "lets staff review search health, inspect tooltips, and drill into terms",time: Time.zone.local(2026, 5, 14, 12, 0, 0) doFabricate.times(- 15,+ 2,:clicked_search_log,term: "ruby",user: user,created_at: "2026-05-10 10:00",)- Fabricate.times(10, :search_log, term: "ruby", user: user, created_at: "2026-05-10 11:00")- Fabricate.times(5, :search_log, term: "ruby", user: user, created_at: "2026-04-20 10:00")+ Fabricate.times(3, :search_log, term: "ruby", user: user, created_at: "2026-05-10 11:00")+ Fabricate(:search_log, term: "ruby", user: user, created_at: "2026-04-20 10:00")- Fabricate.times(- 2,+ Fabricate(:clicked_search_log,term: "markdown tables",user: user,created_at: "2026-04-20 11:00",)-- Fabricate.times(- 13,- :search_log,- term: "markdown tables",- user: user,- created_at: "2026-04-20 12:00",- )Fabricate.times(3,:search_log,term: "markdown tables",user: user,- created_at: "2026-05-03 10:00",+ created_at: "2026-04-20 12:00",)+ Fabricate(:search_log, term: "markdown tables", user: user, created_at: "2026-05-03 10:00")- Fabricate.times(2, :search_log, term: "discobot", user: user, created_at: "2026-05-10 12:00")+ Fabricate(:search_log, term: "discobot", user: user, created_at: "2026-04-20 13:00")Fabricate.times(- 20,+ 6,:clicked_search_log,term: "ruby",user: user,created_at: "2026-03-20 10:00",)- Fabricate.times(20, :search_log, term: "ruby", user: user, created_at: "2026-03-20 11:00")Fabricate(:search_log, term: "admin-search", user: current_user, created_at: "2026-05-10 08:00")Fabricate(:clicked_search_log, term: "ruby", user: moderator, created_at: "2026-05-10 08:30")@@ -77,10 +69,22 @@ describe "Admin Dashboard Redesign | Search section" docreated_at: "2026-03-20 08:00",)- # Anonymous searches (likely crawlers) must be excluded from every metric. If they+ # Searches flagged as crawler traffic must be excluded from every metric. If they# were counted, "crawlerbot" would top trending and the no-result rate would spike.- Fabricate.times(100, :search_log, term: "crawlerbot", created_at: "2026-05-10 09:00")- Fabricate.times(50, :clicked_search_log, term: "ruby", created_at: "2026-05-10 09:30")+ Fabricate.times(+ 10,+ :search_log,+ term: "crawlerbot",+ crawler: true,+ created_at: "2026-05-10 09:00",+ )+ Fabricate.times(+ 2,+ :clicked_search_log,+ term: "ruby",+ crawler: true,+ created_at: "2026-05-10 09:30",+ )dashboard.visitexpect(dashboard).to have_section("search")@@ -93,12 +97,12 @@ describe "Admin Dashboard Redesign | Search section" do"increased. Review the content gaps to see what's missing.",)- expect(search).to have_total_searches_kpi("50", improving_delta: "+25%")- expect(search).to have_no_result_rate_kpi("4%", worsening_delta: "+4%")+ expect(search).to have_total_searches_kpi("12", improving_delta: "+100%")+ expect(search).to have_no_result_rate_kpi("8%", worsening_delta: "+8%")search.hover_total_searches_tooltipexpect(search).to have_total_searches_tooltip(- "The number of searches performed by members in your community.",+ "The number of searches performed in your community.",)search.hover_no_result_rate_tooltip@@ -108,13 +112,13 @@ describe "Admin Dashboard Redesign | Search section" do)search.hover_trending_tooltip- expect(search).to have_trending_tooltip("The most popular search terms among members.")+ expect(search).to have_trending_tooltip("The most popular search terms in your community.")expect(search).to have_trending_rows([- { term: "ruby", searches: 30 },- { term: "markdown tables", searches: 18 },- { term: "discobot", searches: 2 },+ { term: "ruby", searches: 6 },+ { term: "markdown tables", searches: 5 },+ { term: "discobot", searches: 1 },],)expect(search).to have_no_trending_term("admin-search")@@ -122,8 +126,8 @@ describe "Admin Dashboard Redesign | Search section" doexpect(search).to have_content_gap_rows([- { term: "markdown tables", searches: 18, badge: "Poor match" },- { term: "discobot", searches: 2, badge: "No match" },+ { term: "markdown tables", searches: 5, badge: "Poor match" },+ { term: "discobot", searches: 1, badge: "No match" },],)@@ -145,14 +149,14 @@ describe "Admin Dashboard Redesign | Search section" do"looking for more often.",)- expect(search).to have_total_searches_kpi("27", improving_delta: "+800%")- expect(search).to have_no_result_rate_kpi("7%", improving_delta: "-93%")+ expect(search).to have_total_searches_kpi("5", improving_delta: "+400%")+ expect(search).to have_no_result_rate_kpi("0%", improving_delta: "-100%")search.click_trending_term("ruby")expect(page).to have_current_path("/admin/logs/search_logs/term", ignore_query: true)expect(Rack::Utils.parse_query(URI.parse(page.current_url).query)).to eq(- "searchType" => "non_staff_only",+ "searchType" => "human_only","period" => "weekly","term" => "ruby",)@@ -181,6 +185,7 @@ describe "Admin Dashboard Redesign | Search section" douser: user,created_at: "2026-05-10 11:00",)+ Fabricate.times(2, :search_log, term: "ghost", created_at: "2026-05-10 12:00")dashboard.visitsearch = dashboard.search@@ -190,8 +195,36 @@ describe "Admin Dashboard Redesign | Search section" do"Members are conducting more searches in your community, but the no-result rate has " \"increased. Review the content gaps to see what's missing.",)- expect(search).to have_total_searches_kpi("10")- expect(search).to have_alert_no_result_rate_kpi("50%")+ expect(search).to have_total_searches_kpi("12")+ expect(search).to have_alert_no_result_rate_kpi("58%")+ end++ it "stays members-only and drills into member searches while crawler detection is disabled",+ time: Time.zone.local(2026, 5, 14, 12, 0, 0) do+ SiteSetting.improved_crawler_detection = false+ Fabricate.times(+ 4,+ :clicked_search_log,+ term: "ruby",+ user: user,+ created_at: "2026-05-10 10:00",+ )+ Fabricate.times(2, :search_log, term: "crawlerbait", created_at: "2026-05-10 11:00")++ dashboard.visit+ search = dashboard.search++ expect(search).to have_total_searches_kpi("4")+ expect(search).to have_trending_rows([{ term: "ruby", searches: 4 }])+ expect(search).to have_no_trending_term("crawlerbait")++ search.click_trending_term("ruby")++ expect(Rack::Utils.parse_query(URI.parse(page.current_url).query)).to eq(+ "searchType" => "non_staff_only",+ "period" => "monthly",+ "term" => "ruby",+ )endit "shows staff a graceful empty state when no searches were logged" do@@ -257,7 +290,7 @@ describe "Admin Dashboard Redesign | Search section" doexpect(page).to have_current_path("/admin/logs/search_logs/term", ignore_query: true)expect(Rack::Utils.parse_query(URI.parse(page.current_url).query)).to eq(- "searchType" => "non_staff_only",+ "searchType" => "human_only","period" => "all","term" => "ruby",)diff --git a/spec/tasks/admin_dashboard_spec.rb b/spec/tasks/admin_dashboard_spec.rbnew file mode 100644index 00000000000..792eb3e09bd--- /dev/null+++ b/spec/tasks/admin_dashboard_spec.rb@@ -0,0 +1,23 @@+# frozen_string_literal: true++describe "admin_dashboard:rebuild_rollups" do+ it "rebuilds every rollup when no name is given" do+ DashboardRollupRebuilder.expects(:rebuild!).with(nil)++ capture_stdout { invoke_rake_task("admin_dashboard:rebuild_rollups") }+ end++ it "rebuilds only the named rollup" do+ DashboardRollupRebuilder.expects(:rebuild!).with("category_activity")++ capture_stdout { invoke_rake_task("admin_dashboard:rebuild_rollups", "category_activity") }+ end++ it "aborts on an unknown rollup name" do+ DashboardRollupRebuilder.expects(:rebuild!).never++ expect { capture_stdout { invoke_rake_task("admin_dashboard:rebuild_rollups", "nope") } }.to(+ raise_error(SystemExit),+ )+ end+end