Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Reclassify likely crawlers in site traffic reporting (#42199)

Krzysztof Kotlarekbf8d011bddd9
diff --git a/app/jobs/scheduled/detect_crawler_pageviews.rb b/app/jobs/scheduled/detect_crawler_pageviews.rb
index c736e0f78b2..ec76c622999 100644
--- a/app/jobs/scheduled/detect_crawler_pageviews.rb
+++ b/app/jobs/scheduled/detect_crawler_pageviews.rb
@@ -8,7 +8,7 @@ module Jobs
WINDOW_DELAY = BrowserPageviewSessionEngagement::BEACON_SETTLE_PERIOD
def execute(args)
- return if !SiteSetting.experimental_detect_crawler_pageviews
+ return if !UpcomingChanges.enabled?(:improved_crawler_detection)
window_end = Time.now - WINDOW_DELAY
CrawlerScorer.score!(window_start: window_end - LOOKBACK, window_end: window_end)
diff --git a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb
index c7a4a68d67d..6b36f7d5da5 100644
--- a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb
+++ b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb
@@ -11,11 +11,31 @@ module Jobs
aggregate_pageviews
aggregate_engagement
+ aggregate_crawlers
backfill_referrers
end
private
+ def aggregate_crawlers
+ return if !UpcomingChanges.enabled?(:improved_crawler_detection)
+
+ start_date, end_date = crawler_aggregation_window
+ return if start_date.nil?
+
+ BrowserPageviewCrawlerDailyRollup.aggregate(start_date: start_date, end_date: end_date)
+ end
+
+ def crawler_aggregation_window
+ end_date = Time.zone.today
+
+ if BrowserPageviewCrawlerDailyRollup.none?
+ [earliest_event_date, end_date]
+ else
+ [1.day.ago.to_date, end_date]
+ end
+ end
+
def aggregate_pageviews
start_date, end_date = pageview_aggregation_window
return if start_date.nil?
@@ -48,17 +68,19 @@ module Jobs
end_date = Time.zone.today
if BrowserPageviewCountryDailyRollup.none? && BrowserPageviewReferrerDailyRollup.none?
- earliest_event_date =
- BrowserPageviewEvent
- .where(BrowserPageviewEvent.rollup_source_condition)
- .minimum(:created_at)
- &.to_date
[earliest_event_date, end_date]
else
[1.day.ago.to_date, end_date]
end
end
+ def earliest_event_date
+ BrowserPageviewEvent
+ .where(BrowserPageviewEvent.rollup_source_condition)
+ .minimum(:created_at)
+ &.to_date
+ end
+
def backfill_referrers
rows = next_batch
return if rows.empty?
diff --git a/app/models/browser_pageview_crawler_daily_rollup.rb b/app/models/browser_pageview_crawler_daily_rollup.rb
new file mode 100644
index 00000000000..127268edc00
--- /dev/null
+++ b/app/models/browser_pageview_crawler_daily_rollup.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class BrowserPageviewCrawlerDailyRollup < ActiveRecord::Base
+ def self.aggregate(start_date:, end_date:)
+ start_date = start_date.to_date
+ end_date = end_date.to_date + 1
+
+ DB.exec(<<~SQL, start_date:, end_date:, threshold: CrawlerScorer::BOT_SCORE_THRESHOLD)
+ INSERT INTO browser_pageview_crawler_daily_rollups (date, logged_in, count)
+ SELECT
+ created_at::date AS date,
+ user_id IS NOT NULL AS logged_in,
+ COUNT(*) AS count
+ FROM browser_pageview_events
+ WHERE created_at >= :start_date
+ AND created_at < :end_date
+ AND score > :threshold
+ AND #{BrowserPageviewEvent.rollup_source_condition}
+ GROUP BY date, logged_in
+ ON CONFLICT (date, logged_in) DO UPDATE
+ SET count = EXCLUDED.count
+ SQL
+ end
+end
+
+# == Schema Information
+#
+# Table name: browser_pageview_crawler_daily_rollups
+#
+# id :bigint not null, primary key
+# count :bigint not null
+# date :date not null
+# logged_in :boolean not null
+#
+# Indexes
+#
+# idx_bpcrawler_rollups_date_logged_in_unique (date,logged_in) UNIQUE
+#
diff --git a/app/models/concerns/reports/site_traffic.rb b/app/models/concerns/reports/site_traffic.rb
index 3e3675fedea..69f3ebe747e 100644
--- a/app/models/concerns/reports/site_traffic.rb
+++ b/app/models/concerns/reports/site_traffic.rb
@@ -7,6 +7,7 @@ module Reports::SiteTraffic
"page_view_logged_in_browser" => "#4B3CE0",
"page_view_anon_browser" => "#9C8DEC",
"page_view_crawler" => "#D5CDF7",
+ "page_view_likely_crawler" => "#B3AAC9",
"page_view_embed" => "#E6E1F8",
"page_view_other" => "#E84A5F",
}.freeze
diff --git a/app/services/admin_dashboard_site_traffic.rb b/app/services/admin_dashboard_site_traffic.rb
index 47f1bd78487..1f9eab972ef 100644
--- a/app/services/admin_dashboard_site_traffic.rb
+++ b/app/services/admin_dashboard_site_traffic.rb
@@ -8,6 +8,7 @@ class AdminDashboardSiteTraffic
anonymous: "page_view_anon_browser",
embedded: "page_view_embed",
crawlers: "page_view_crawler",
+ likely_crawlers: "page_view_likely_crawler",
}.freeze
private_constant :DEFAULT_RANGE_DAYS
private_constant :TOP_CARD_LIMIT
@@ -93,15 +94,20 @@ class AdminDashboardSiteTraffic
def series_ids(include_embedded:)
series = %i[logged_in]
- return series if login_required?
-
- series << :anonymous
- series << :embedded if include_embedded
- series << :crawlers
+ series << :anonymous if !login_required?
+ series << :embedded if !login_required? && include_embedded
+ series << :likely_crawlers if likely_crawlers_enabled?
+ series << :crawlers if !login_required?
series
end
+ def likely_crawlers_enabled?
+ return @likely_crawlers_enabled if defined?(@likely_crawlers_enabled)
+
+ @likely_crawlers_enabled = UpcomingChanges.enabled?(:improved_crawler_detection)
+ end
+
def kpis(totals, prior_rows)
kpis = { browser_pageviews: browser_pageviews_kpi(totals, prior_rows) }
logged_in_share = logged_in_share_value(totals)
@@ -199,7 +205,7 @@ class AdminDashboardSiteTraffic
end
def series_req(id)
- selected_request_type_names.fetch(id)
+ selected_request_type_names.fetch(id) { series_label_req(id) }
end
def series_label(id)
@@ -312,31 +318,52 @@ class AdminDashboardSiteTraffic
CAST(:end_date AS date),
INTERVAL '1 day'
) request_date
+ ),
+ likely_crawlers AS (
+ SELECT
+ date,
+ COALESCE(SUM(count) FILTER (WHERE logged_in), 0)::bigint AS logged_in,
+ COALESCE(SUM(count) FILTER (WHERE NOT logged_in), 0)::bigint AS anonymous
+ FROM browser_pageview_crawler_daily_rollups
+ WHERE :likely_crawlers_enabled
+ AND date >= CAST(:start_date AS date)
+ AND date <= CAST(:end_date AS date)
+ GROUP BY date
)
SELECT
dates.date,
- COALESCE(
- SUM(
- CASE
- WHEN dates.date < :beacon_cutover_date AND ar.req_type = :logged_in_req_type THEN ar.count
- WHEN dates.date >= :beacon_cutover_date AND ar.req_type = :logged_in_beacon_req_type THEN ar.count
- ELSE 0
- END
- ),
- 0
+ GREATEST(
+ 0,
+ COALESCE(
+ SUM(
+ CASE
+ WHEN dates.date < :beacon_cutover_date AND ar.req_type = :logged_in_req_type THEN ar.count
+ WHEN dates.date >= :beacon_cutover_date AND ar.req_type = :logged_in_beacon_req_type THEN ar.count
+ ELSE 0
+ END
+ ),
+ 0
+ ) - COALESCE(MAX(lc.logged_in), 0)
)::bigint AS logged_in,
- COALESCE(
- SUM(
- CASE
- WHEN dates.date < :beacon_cutover_date AND ar.req_type = :anonymous_req_type THEN ar.count
- WHEN dates.date >= :beacon_cutover_date AND ar.req_type = :anonymous_beacon_req_type THEN ar.count
- ELSE 0
- END
- ),
- 0
+ GREATEST(
+ 0,
+ COALESCE(
+ SUM(
+ CASE
+ WHEN dates.date < :beacon_cutover_date AND ar.req_type = :anonymous_req_type THEN ar.count
+ WHEN dates.date >= :beacon_cutover_date AND ar.req_type = :anonymous_beacon_req_type THEN ar.count
+ ELSE 0
+ END
+ ),
+ 0
+ ) - COALESCE(MAX(lc.anonymous), 0)
)::bigint AS anonymous,
COALESCE(SUM(CASE WHEN ar.req_type = :crawler_req_type THEN ar.count ELSE 0 END), 0)::bigint AS crawlers,
- COALESCE(SUM(CASE WHEN ar.req_type = :embedded_req_type THEN ar.count ELSE 0 END), 0)::bigint AS embedded
+ COALESCE(SUM(CASE WHEN ar.req_type = :embedded_req_type THEN ar.count ELSE 0 END), 0)::bigint AS embedded,
+ (
+ COALESCE(MAX(lc.logged_in), 0)
+ + CASE WHEN :login_required THEN 0 ELSE COALESCE(MAX(lc.anonymous), 0) END
+ )::bigint AS likely_crawlers
FROM dates
LEFT JOIN application_requests ar
ON ar.date = dates.date
@@ -348,11 +375,14 @@ class AdminDashboardSiteTraffic
:crawler_req_type,
:embedded_req_type
)
+ LEFT JOIN likely_crawlers lc ON lc.date = dates.date
GROUP BY dates.date
ORDER BY dates.date ASC
SQL
start_date: range_start_date,
end_date: range_end_date,
+ likely_crawlers_enabled: likely_crawlers_enabled?,
+ login_required: login_required?,
beacon_cutover_date: cutover_date,
logged_in_req_type: selected_request_types[:logged_in],
anonymous_req_type: selected_request_types[:anonymous],
@@ -374,6 +404,7 @@ class AdminDashboardSiteTraffic
anonymous: anonymous,
embedded: embedded,
crawlers: crawlers,
+ likely_crawlers: likely_crawlers_enabled? ? sum_rows(rows, :likely_crawlers) : 0,
human: logged_in + anonymous,
}
end
diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml
index ad4f3421a63..e331933eec3 100644
--- a/config/locales/server.en.yml
+++ b/config/locales/server.en.yml
@@ -1518,6 +1518,7 @@ en:
page_view_anon_browser: "Anonymous"
page_view_logged_in_browser: "Logged in"
page_view_crawler: "Crawlers"
+ page_view_likely_crawler: "Likely crawlers"
page_view_embed: "Embedded pageviews"
page_view_other: "Other traffic"
yaxis: "Day"
@@ -2980,6 +2981,7 @@ en:
enable_horizon_high_context_topic_cards: "Shows richer topic cards in the Horizon theme, including topic excerpts, tags, and plugin extended information."
granular_anonymous_and_logged_in_groups_permissions: "Makes group-based site setting permissions treat 'everyone' as 'all logged-in users', and enables the new 'anonymous_users' and 'logged_in_users' pseudogroups so admins can explicitly target anonymous visitors and authenticated users."
dashboard_improvements: "Enables the redesigned admin dashboard that lets you pin custom reports and includes improved site traffic, engagement, and other analytics tied to community success and value."
+ improved_crawler_detection: "Enables improved crawler tracking which can better detect likely crawlers and ensure traffic reporting is more accurate."
enable_invite_modal_with_roles: "Enables a redesigned invite flow that lets admins invite someone directly as an admin. The invitee becomes a moderator on signup and an admin once the inviter confirms via email."
update_pending_users_reminder_default: "We are updating the default for the {{setting:pending_users_reminder_delay_minutes}} setting to 30 minutes so that staff are promptly notified when a new member is waiting for approval. If you have previously changed this setting, this change will have no effect."
diff --git a/config/site_settings.yml b/config/site_settings.yml
index 56713fcc1a5..2a458143c97 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -653,9 +653,6 @@ basic:
max: 86400
hidden: true
client: true
- experimental_detect_crawler_pageviews:
- default: false
- hidden: true
crawler_automation_user_agents:
default: "HeadlessChrome|Playwright|Puppeteer|Selenium|PhantomJS|jsdom"
hidden: true
@@ -4830,6 +4827,15 @@ experimental:
status: "beta"
impact: "feature,admins"
learn_more_url: "https://meta.discourse.org/t/-/404528"
+ improved_crawler_detection:
+ default: false
+ hidden: true
+ client: true
+ upcoming_change:
+ status: "conceptual"
+ impact: "other,all_members"
+ allow_enabled_for:
+ - everyone
admin_dashboard_reports_seeded:
default: false
hidden: true
diff --git a/db/migrate/20260728033946_create_browser_pageview_crawler_daily_rollups.rb b/db/migrate/20260728033946_create_browser_pageview_crawler_daily_rollups.rb
new file mode 100644
index 00000000000..37e5f42cc64
--- /dev/null
+++ b/db/migrate/20260728033946_create_browser_pageview_crawler_daily_rollups.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class CreateBrowserPageviewCrawlerDailyRollups < ActiveRecord::Migration[8.0]
+ def change
+ create_table :browser_pageview_crawler_daily_rollups do |t|
+ t.date :date, null: false
+ t.boolean :logged_in, null: false
+ t.bigint :count, null: false
+ end
+
+ add_index :browser_pageview_crawler_daily_rollups,
+ %i[date logged_in],
+ unique: true,
+ name: "idx_bpcrawler_rollups_date_logged_in_unique"
+ end
+end
diff --git a/db/post_migrate/20260728050038_remove_experimental_detect_crawler_pageviews_setting.rb b/db/post_migrate/20260728050038_remove_experimental_detect_crawler_pageviews_setting.rb
new file mode 100644
index 00000000000..36919853379
--- /dev/null
+++ b/db/post_migrate/20260728050038_remove_experimental_detect_crawler_pageviews_setting.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+class RemoveExperimentalDetectCrawlerPageviewsSetting < ActiveRecord::Migration[8.0]
+ def up
+ execute "DELETE FROM site_settings WHERE name = 'experimental_detect_crawler_pageviews'"
+ end
+
+ def down
+ raise ActiveRecord::IrreversibleMigration
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index e6f2aa9982f..6459af85a3f 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1978,6 +1978,37 @@ CREATE SEQUENCE public.browser_pageview_country_daily_rollups_id_seq
ALTER SEQUENCE public.browser_pageview_country_daily_rollups_id_seq OWNED BY public.browser_pageview_country_daily_rollups.id;
+--
+-- Name: browser_pageview_crawler_daily_rollups; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.browser_pageview_crawler_daily_rollups (
+ id bigint NOT NULL,
+ date date NOT NULL,
+ logged_in boolean NOT NULL,
+ count bigint NOT NULL
+);
+
+
+--
+-- Name: browser_pageview_crawler_daily_rollups_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.browser_pageview_crawler_daily_rollups_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+--
+-- Name: browser_pageview_crawler_daily_rollups_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.browser_pageview_crawler_daily_rollups_id_seq OWNED BY public.browser_pageview_crawler_daily_rollups.id;
+
+
--
-- Name: browser_pageview_event_scores; Type: TABLE; Schema: public; Owner: -
--
@@ -12760,6 +12791,13 @@ ALTER TABLE ONLY public.bookmarks ALTER COLUMN id SET DEFAULT nextval('public.bo
ALTER TABLE ONLY public.browser_pageview_country_daily_rollups ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_country_daily_rollups_id_seq'::regclass);
+--
+-- Name: browser_pageview_crawler_daily_rollups id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.browser_pageview_crawler_daily_rollups ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_crawler_daily_rollups_id_seq'::regclass);
+
+
--
-- Name: browser_pageview_event_scores id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -15053,6 +15091,14 @@ ALTER TABLE ONLY public.browser_pageview_country_daily_rollups
ADD CONSTRAINT browser_pageview_country_daily_rollups_pkey PRIMARY KEY (id);
+--
+-- Name: browser_pageview_crawler_daily_rollups browser_pageview_crawler_daily_rollups_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.browser_pageview_crawler_daily_rollups
+ ADD CONSTRAINT browser_pageview_crawler_daily_rollups_pkey PRIMARY KEY (id);
+
+
--
-- Name: browser_pageview_event_scores browser_pageview_event_scores_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -17529,6 +17575,13 @@ CREATE UNIQUE INDEX idx_bookmarks_user_polymorphic_unique ON public.bookmarks US
CREATE UNIQUE INDEX idx_bpcd_rollups_date_country_unique ON public.browser_pageview_country_daily_rollups USING btree (date, country_code) NULLS NOT DISTINCT;
+--
+-- Name: idx_bpcrawler_rollups_date_logged_in_unique; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX idx_bpcrawler_rollups_date_logged_in_unique ON public.browser_pageview_crawler_daily_rollups USING btree (date, logged_in);
+
+
--
-- Name: idx_bpe_created_at_country_code; Type: INDEX; Schema: public; Owner: -
--
@@ -23082,7 +23135,9 @@ INSERT INTO "schema_migrations" (version) VALUES
('20260728162516'),
('20260728134532'),
('20260728071552'),
+('20260728050038'),
('20260728045008'),
+('20260728033946'),
('20260727085824'),
('20260727035337'),
('20260723183001'),
diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rb
index 05aa6507913..2f6c2044246 100644
--- a/lib/crawler_scorer.rb
+++ b/lib/crawler_scorer.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
class CrawlerScorer
- BOT_SCORE_THRESHOLD = 60
+ BOT_SCORE_THRESHOLD = 55
AUTOMATION_UA_SCORE = 100
diff --git a/spec/fabricators/browser_pageview_crawler_daily_rollup_fabricator.rb b/spec/fabricators/browser_pageview_crawler_daily_rollup_fabricator.rb
new file mode 100644
index 00000000000..cfdf0a94054
--- /dev/null
+++ b/spec/fabricators/browser_pageview_crawler_daily_rollup_fabricator.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+Fabricator(:browser_pageview_crawler_daily_rollup) do
+ date { Date.current }
+ logged_in false
+ count 1
+end
diff --git a/spec/jobs/detect_crawler_pageviews_spec.rb b/spec/jobs/detect_crawler_pageviews_spec.rb
index dde737a08ee..40abf4e3401 100644
--- a/spec/jobs/detect_crawler_pageviews_spec.rb
+++ b/spec/jobs/detect_crawler_pageviews_spec.rb
@@ -1,15 +1,15 @@
# frozen_string_literal: true
RSpec.describe Jobs::DetectCrawlerPageviews do
- it "does nothing when detection is disabled" do
- SiteSetting.experimental_detect_crawler_pageviews = false
+ it "does nothing when the improved crawler detection change is disabled" do
+ SiteSetting.improved_crawler_detection = false
CrawlerScorer.expects(:score!).never
described_class.new.execute({})
end
it "scores a one hour wide window, held back by the beacon settle period" do
- SiteSetting.experimental_detect_crawler_pageviews = true
+ SiteSetting.improved_crawler_detection = true
freeze_time
CrawlerScorer.expects(:score!).with(window_start: 90.minutes.ago, window_end: 30.minutes.ago)
diff --git a/spec/jobs/maintain_browser_pageview_rollups_spec.rb b/spec/jobs/maintain_browser_pageview_rollups_spec.rb
index b62f225cdaa..a86fe5fafcd 100644
--- a/spec/jobs/maintain_browser_pageview_rollups_spec.rb
+++ b/spec/jobs/maintain_browser_pageview_rollups_spec.rb
@@ -110,6 +110,70 @@ RSpec.describe Jobs::MaintainBrowserPageviewRollups do
end
end
+ context "when aggregating crawler rollups" do
+ let(:above_threshold) { CrawlerScorer::BOT_SCORE_THRESHOLD + 1 }
+
+ before { freeze_time(Time.utc(2026, 6, 20, 12, 0, 0)) }
+
+ it "does nothing when improved_crawler_detection is disabled" do
+ Fabricate(
+ :browser_pageview_event,
+ score: above_threshold,
+ created_at: Time.utc(2026, 6, 20, 9),
+ )
+
+ job.execute({})
+
+ expect(BrowserPageviewCrawlerDailyRollup.count).to eq(0)
+ end
+
+ context "when improved_crawler_detection is enabled" do
+ before { SiteSetting.improved_crawler_detection = true }
+
+ it "backfills the full event history on the first run" do
+ Fabricate(
+ :browser_pageview_event,
+ score: above_threshold,
+ created_at: Time.utc(2026, 5, 1, 9),
+ )
+ Fabricate(
+ :browser_pageview_event,
+ score: above_threshold,
+ created_at: Time.utc(2026, 6, 20, 9),
+ )
+
+ job.execute({})
+
+ expect(BrowserPageviewCrawlerDailyRollup.sum(:count)).to eq(2)
+ end
+
+ it "only refreshes yesterday and today once history has been rolled up" do
+ Fabricate(
+ :browser_pageview_event,
+ score: above_threshold,
+ created_at: Time.utc(2026, 6, 20, 9),
+ )
+ job.execute({})
+
+ Fabricate(
+ :browser_pageview_event,
+ score: above_threshold,
+ created_at: Time.utc(2026, 5, 1, 9),
+ )
+ Fabricate(
+ :browser_pageview_event,
+ score: above_threshold,
+ created_at: Time.utc(2026, 6, 19, 9),
+ )
+ job.execute({})
+
+ expect(
+ BrowserPageviewCrawlerDailyRollup.order(:date).pluck(:date, :count),
+ ).to contain_exactly([Date.new(2026, 6, 19), 1], [Date.new(2026, 6, 20), 1])
+ end
+ end
+ end
+
context "when aggregating engagement rollups" do
before { freeze_time(Time.utc(2026, 6, 20, 12, 0, 0)) }
diff --git a/spec/models/browser_pageview_crawler_daily_rollup_spec.rb b/spec/models/browser_pageview_crawler_daily_rollup_spec.rb
new file mode 100644
index 00000000000..755348f5668
--- /dev/null
+++ b/spec/models/browser_pageview_crawler_daily_rollup_spec.rb
@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+RSpec.describe BrowserPageviewCrawlerDailyRollup do
+ describe ".aggregate" do
+ let(:start_date) { 3.days.ago.to_date }
+ let(:end_date) { Date.current }
+ let(:above_threshold) { CrawlerScorer::BOT_SCORE_THRESHOLD + 1 }
+
+ it "groups likely crawler events by date and logged in state" do
+ user = Fabricate(:user)
+ yesterday = 1.day.ago
+ today = Time.current
+
+ Fabricate(:browser_pageview_event, score: above_threshold, created_at: yesterday)
+ Fabricate(:browser_pageview_event, score: above_threshold, created_at: today)
+ Fabricate(
+ :browser_pageview_event,
+ score: above_threshold,
+ user_id: user.id,
+ created_at: today,
+ )
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ expect(described_class.order(:date, :logged_in).pluck(:date, :logged_in, :count)).to eq(
+ [[yesterday.to_date, false, 1], [today.to_date, false, 1], [today.to_date, true, 1]],
+ )
+ end
+
+ it "ignores events at or below the threshold, and unscored events" do
+ Fabricate(:browser_pageview_event, score: CrawlerScorer::BOT_SCORE_THRESHOLD)
+ Fabricate(:browser_pageview_event, score: 0)
+ Fabricate(:browser_pageview_event, score: nil)
+ Fabricate(:browser_pageview_event, score: above_threshold)
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ expect(described_class.sum(:count)).to eq(1)
+ end
+
+ it "only aggregates events within the requested date range" do
+ Fabricate(:browser_pageview_event, score: above_threshold, created_at: 10.days.ago)
+ Fabricate(:browser_pageview_event, score: above_threshold, created_at: 1.day.ago)
+
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ expect(described_class.sum(:count)).to eq(1)
+ end
+
+ it "is idempotent and refreshes counts when re-aggregating" do
+ Fabricate(:browser_pageview_event, score: above_threshold)
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ Fabricate(:browser_pageview_event, score: above_threshold)
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+
+ expect(described_class.count).to eq(1)
+ expect(described_class.sum(:count)).to eq(2)
+ end
+
+ it "is a no-op when no likely crawler events exist in the range" do
+ Fabricate(:browser_pageview_event, score: nil)
+
+ expect {
+ described_class.aggregate(start_date: start_date, end_date: end_date)
+ }.not_to change { described_class.count }
+ end
+ end
+end
diff --git a/spec/services/admin_dashboard_site_traffic_spec.rb b/spec/services/admin_dashboard_site_traffic_spec.rb
index 3aec552db7f..dfd0ca8892b 100644
--- a/spec/services/admin_dashboard_site_traffic_spec.rb
+++ b/spec/services/admin_dashboard_site_traffic_spec.rb
@@ -34,6 +34,7 @@ RSpec.describe AdminDashboardSiteTraffic do
anonymous: "page_view_anon_browser",
embedded: "page_view_embed",
crawlers: "page_view_crawler",
+ likely_crawlers: "page_view_likely_crawler",
}.fetch(id)
end
@@ -1085,5 +1086,131 @@ RSpec.describe AdminDashboardSiteTraffic do
)
end
end
+
+ context "for likely crawlers" do
+ before { SiteSetting.improved_crawler_detection = true }
+
+ it "orders likely crawlers ahead of known crawlers" do
+ SiteSetting.embed_topics_list = true
+ Fabricate(:embeddable_host)
+
+ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")
+
+ expect(response[:pageview_series].map { |series| series[:req] }).to eq(
+ %w[
+ page_view_logged_in_browser
+ page_view_anon_browser
+ page_view_embed
+ page_view_likely_crawler
+ page_view_crawler
+ ],
+ )
+ end
+
+ it "reclassifies likely crawlers out of the logged in and anonymous series" do
+ Fabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 10)
+ Fabricate(:anonymous_browser_application_request, date: "2026-05-01", count: 20)
+
+ Fabricate(
+ :browser_pageview_crawler_daily_rollup,
+ date: Date.new(2026, 5, 1),
+ logged_in: true,
+ count: 4,
+ )
+ Fabricate(
+ :browser_pageview_crawler_daily_rollup,
+ date: Date.new(2026, 5, 1),
+ logged_in: false,
+ count: 15,
+ )
+
+ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")
+
+ expect(traffic_series_data(response, :logged_in)).to eq([traffic_point("2026-05-01", 6)])
+ expect(traffic_series_data(response, :anonymous)).to eq([traffic_point("2026-05-01", 5)])
+ expect(traffic_series_data(response, :likely_crawlers)).to eq(
+ [traffic_point("2026-05-01", 19)],
+ )
+ expect(response[:kpis][:browser_pageviews][:value]).to eq(11)
+ end
+
+ it "never drives a series below zero when the rollup exceeds recorded pageviews" do
+ Fabricate(:anonymous_browser_application_request, date: "2026-05-01", count: 2)
+ Fabricate(
+ :browser_pageview_crawler_daily_rollup,
+ date: Date.new(2026, 5, 1),
+ logged_in: false,
+ count: 9,
+ )
+
+ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")
+
+ expect(traffic_series_data(response, :anonymous)).to eq([traffic_point("2026-05-01", 0)])
+ expect(traffic_series_data(response, :likely_crawlers)).to eq(
+ [traffic_point("2026-05-01", 9)],
+ )
+ end
+
+ it "ignores rollups outside the requested range" do
+ Fabricate(:anonymous_browser_application_request, date: "2026-05-01", count: 10)
+ Fabricate(
+ :browser_pageview_crawler_daily_rollup,
+ date: Date.new(2026, 4, 20),
+ logged_in: false,
+ count: 7,
+ )
+
+ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")
+
+ expect(traffic_series_data(response, :anonymous)).to eq([traffic_point("2026-05-01", 10)])
+ expect(traffic_series_data(response, :likely_crawlers)).to eq(
+ [traffic_point("2026-05-01", 0)],
+ )
+ end
+
+ it "excludes anonymous crawlers when login is required" do
+ SiteSetting.login_required = true
+
+ Fabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 10)
+ Fabricate(
+ :browser_pageview_crawler_daily_rollup,
+ date: Date.new(2026, 5, 1),
+ logged_in: true,
+ count: 3,
+ )
+ Fabricate(
+ :browser_pageview_crawler_daily_rollup,
+ date: Date.new(2026, 5, 1),
+ logged_in: false,
+ count: 8,
+ )
+
+ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")
+
+ expect(traffic_series_data(response, :logged_in)).to eq([traffic_point("2026-05-01", 7)])
+ expect(traffic_series_data(response, :likely_crawlers)).to eq(
+ [traffic_point("2026-05-01", 3)],
+ )
+ end
+
+ it "leaves the series out and keeps counts intact when the change is disabled" do
+ SiteSetting.improved_crawler_detection = false
+
+ Fabricate(:anonymous_browser_application_request, date: "2026-05-01", count: 10)
+ Fabricate(
+ :browser_pageview_crawler_daily_rollup,
+ date: Date.new(2026, 5, 1),
+ logged_in: false,
+ count: 6,
+ )
+
+ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")
+
+ expect(traffic_series_data(response, :anonymous)).to eq([traffic_point("2026-05-01", 10)])
+ expect(response[:pageview_series].map { |series| series[:req] }).not_to include(
+ "page_view_likely_crawler",
+ )
+ end
+ end
end
end