A platform for community discussion. Free, open, simple.
FEATURE: Reclassify likely crawlers in site traffic reporting (#42199)
Krzysztof Kotlarekbf8d011bddd9
diff --git a/app/jobs/scheduled/detect_crawler_pageviews.rb b/app/jobs/scheduled/detect_crawler_pageviews.rbindex c736e0f78b2..ec76c622999 100644--- a/app/jobs/scheduled/detect_crawler_pageviews.rb+++ b/app/jobs/scheduled/detect_crawler_pageviews.rb@@ -8,7 +8,7 @@ module JobsWINDOW_DELAY = BrowserPageviewSessionEngagement::BEACON_SETTLE_PERIODdef execute(args)- return if !SiteSetting.experimental_detect_crawler_pageviews+ return if !UpcomingChanges.enabled?(:improved_crawler_detection)window_end = Time.now - WINDOW_DELAYCrawlerScorer.score!(window_start: window_end - LOOKBACK, window_end: window_end)diff --git a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb b/app/jobs/scheduled/maintain_browser_pageview_rollups.rbindex c7a4a68d67d..6b36f7d5da5 100644--- a/app/jobs/scheduled/maintain_browser_pageview_rollups.rb+++ b/app/jobs/scheduled/maintain_browser_pageview_rollups.rb@@ -11,11 +11,31 @@ module Jobsaggregate_pageviewsaggregate_engagement+ aggregate_crawlersbackfill_referrersendprivate+ def aggregate_crawlers+ return if !UpcomingChanges.enabled?(:improved_crawler_detection)++ start_date, end_date = crawler_aggregation_window+ return if start_date.nil?++ BrowserPageviewCrawlerDailyRollup.aggregate(start_date: start_date, end_date: end_date)+ end++ def crawler_aggregation_window+ end_date = Time.zone.today++ if BrowserPageviewCrawlerDailyRollup.none?+ [earliest_event_date, end_date]+ else+ [1.day.ago.to_date, end_date]+ end+ end+def aggregate_pageviewsstart_date, end_date = pageview_aggregation_windowreturn if start_date.nil?@@ -48,17 +68,19 @@ module Jobsend_date = Time.zone.todayif BrowserPageviewCountryDailyRollup.none? && BrowserPageviewReferrerDailyRollup.none?- earliest_event_date =- BrowserPageviewEvent- .where(BrowserPageviewEvent.rollup_source_condition)- .minimum(:created_at)- &.to_date[earliest_event_date, end_date]else[1.day.ago.to_date, end_date]endend+ def earliest_event_date+ BrowserPageviewEvent+ .where(BrowserPageviewEvent.rollup_source_condition)+ .minimum(:created_at)+ &.to_date+ end+def backfill_referrersrows = next_batchreturn if rows.empty?diff --git a/app/models/browser_pageview_crawler_daily_rollup.rb b/app/models/browser_pageview_crawler_daily_rollup.rbnew file mode 100644index 00000000000..127268edc00--- /dev/null+++ b/app/models/browser_pageview_crawler_daily_rollup.rb@@ -0,0 +1,38 @@+# frozen_string_literal: true++class BrowserPageviewCrawlerDailyRollup < ActiveRecord::Base+ def self.aggregate(start_date:, end_date:)+ start_date = start_date.to_date+ end_date = end_date.to_date + 1++ DB.exec(<<~SQL, start_date:, end_date:, threshold: CrawlerScorer::BOT_SCORE_THRESHOLD)+ INSERT INTO browser_pageview_crawler_daily_rollups (date, logged_in, count)+ SELECT+ created_at::date AS date,+ user_id IS NOT NULL AS logged_in,+ COUNT(*) AS count+ FROM browser_pageview_events+ WHERE created_at >= :start_date+ AND created_at < :end_date+ AND score > :threshold+ AND #{BrowserPageviewEvent.rollup_source_condition}+ GROUP BY date, logged_in+ ON CONFLICT (date, logged_in) DO UPDATE+ SET count = EXCLUDED.count+ SQL+ end+end++# == Schema Information+#+# Table name: browser_pageview_crawler_daily_rollups+#+# id :bigint not null, primary key+# count :bigint not null+# date :date not null+# logged_in :boolean not null+#+# Indexes+#+# idx_bpcrawler_rollups_date_logged_in_unique (date,logged_in) UNIQUE+#diff --git a/app/models/concerns/reports/site_traffic.rb b/app/models/concerns/reports/site_traffic.rbindex 3e3675fedea..69f3ebe747e 100644--- a/app/models/concerns/reports/site_traffic.rb+++ b/app/models/concerns/reports/site_traffic.rb@@ -7,6 +7,7 @@ module Reports::SiteTraffic"page_view_logged_in_browser" => "#4B3CE0","page_view_anon_browser" => "#9C8DEC","page_view_crawler" => "#D5CDF7",+ "page_view_likely_crawler" => "#B3AAC9","page_view_embed" => "#E6E1F8","page_view_other" => "#E84A5F",}.freezediff --git a/app/services/admin_dashboard_site_traffic.rb b/app/services/admin_dashboard_site_traffic.rbindex 47f1bd78487..1f9eab972ef 100644--- a/app/services/admin_dashboard_site_traffic.rb+++ b/app/services/admin_dashboard_site_traffic.rb@@ -8,6 +8,7 @@ class AdminDashboardSiteTrafficanonymous: "page_view_anon_browser",embedded: "page_view_embed",crawlers: "page_view_crawler",+ likely_crawlers: "page_view_likely_crawler",}.freezeprivate_constant :DEFAULT_RANGE_DAYSprivate_constant :TOP_CARD_LIMIT@@ -93,15 +94,20 @@ class AdminDashboardSiteTrafficdef series_ids(include_embedded:)series = %i[logged_in]- return series if login_required?-- series << :anonymous- series << :embedded if include_embedded- series << :crawlers+ series << :anonymous if !login_required?+ series << :embedded if !login_required? && include_embedded+ series << :likely_crawlers if likely_crawlers_enabled?+ series << :crawlers if !login_required?seriesend+ def likely_crawlers_enabled?+ return @likely_crawlers_enabled if defined?(@likely_crawlers_enabled)++ @likely_crawlers_enabled = UpcomingChanges.enabled?(:improved_crawler_detection)+ end+def kpis(totals, prior_rows)kpis = { browser_pageviews: browser_pageviews_kpi(totals, prior_rows) }logged_in_share = logged_in_share_value(totals)@@ -199,7 +205,7 @@ class AdminDashboardSiteTrafficenddef series_req(id)- selected_request_type_names.fetch(id)+ selected_request_type_names.fetch(id) { series_label_req(id) }enddef series_label(id)@@ -312,31 +318,52 @@ class AdminDashboardSiteTrafficCAST(:end_date AS date),INTERVAL '1 day') request_date+ ),+ likely_crawlers AS (+ SELECT+ date,+ COALESCE(SUM(count) FILTER (WHERE logged_in), 0)::bigint AS logged_in,+ COALESCE(SUM(count) FILTER (WHERE NOT logged_in), 0)::bigint AS anonymous+ FROM browser_pageview_crawler_daily_rollups+ WHERE :likely_crawlers_enabled+ AND date >= CAST(:start_date AS date)+ AND date <= CAST(:end_date AS date)+ GROUP BY date)SELECTdates.date,- COALESCE(- SUM(- CASE- WHEN dates.date < :beacon_cutover_date AND ar.req_type = :logged_in_req_type THEN ar.count- WHEN dates.date >= :beacon_cutover_date AND ar.req_type = :logged_in_beacon_req_type THEN ar.count- ELSE 0- END- ),- 0+ GREATEST(+ 0,+ COALESCE(+ SUM(+ CASE+ WHEN dates.date < :beacon_cutover_date AND ar.req_type = :logged_in_req_type THEN ar.count+ WHEN dates.date >= :beacon_cutover_date AND ar.req_type = :logged_in_beacon_req_type THEN ar.count+ ELSE 0+ END+ ),+ 0+ ) - COALESCE(MAX(lc.logged_in), 0))::bigint AS logged_in,- COALESCE(- SUM(- CASE- WHEN dates.date < :beacon_cutover_date AND ar.req_type = :anonymous_req_type THEN ar.count- WHEN dates.date >= :beacon_cutover_date AND ar.req_type = :anonymous_beacon_req_type THEN ar.count- ELSE 0- END- ),- 0+ GREATEST(+ 0,+ COALESCE(+ SUM(+ CASE+ WHEN dates.date < :beacon_cutover_date AND ar.req_type = :anonymous_req_type THEN ar.count+ WHEN dates.date >= :beacon_cutover_date AND ar.req_type = :anonymous_beacon_req_type THEN ar.count+ ELSE 0+ END+ ),+ 0+ ) - COALESCE(MAX(lc.anonymous), 0))::bigint AS anonymous,COALESCE(SUM(CASE WHEN ar.req_type = :crawler_req_type THEN ar.count ELSE 0 END), 0)::bigint AS crawlers,- COALESCE(SUM(CASE WHEN ar.req_type = :embedded_req_type THEN ar.count ELSE 0 END), 0)::bigint AS embedded+ COALESCE(SUM(CASE WHEN ar.req_type = :embedded_req_type THEN ar.count ELSE 0 END), 0)::bigint AS embedded,+ (+ COALESCE(MAX(lc.logged_in), 0)+ + CASE WHEN :login_required THEN 0 ELSE COALESCE(MAX(lc.anonymous), 0) END+ )::bigint AS likely_crawlersFROM datesLEFT JOIN application_requests arON ar.date = dates.date@@ -348,11 +375,14 @@ class AdminDashboardSiteTraffic:crawler_req_type,:embedded_req_type)+ LEFT JOIN likely_crawlers lc ON lc.date = dates.dateGROUP BY dates.dateORDER BY dates.date ASCSQLstart_date: range_start_date,end_date: range_end_date,+ likely_crawlers_enabled: likely_crawlers_enabled?,+ login_required: login_required?,beacon_cutover_date: cutover_date,logged_in_req_type: selected_request_types[:logged_in],anonymous_req_type: selected_request_types[:anonymous],@@ -374,6 +404,7 @@ class AdminDashboardSiteTrafficanonymous: anonymous,embedded: embedded,crawlers: crawlers,+ likely_crawlers: likely_crawlers_enabled? ? sum_rows(rows, :likely_crawlers) : 0,human: logged_in + anonymous,}enddiff --git a/config/locales/server.en.yml b/config/locales/server.en.ymlindex ad4f3421a63..e331933eec3 100644--- a/config/locales/server.en.yml+++ b/config/locales/server.en.yml@@ -1518,6 +1518,7 @@ en:page_view_anon_browser: "Anonymous"page_view_logged_in_browser: "Logged in"page_view_crawler: "Crawlers"+ page_view_likely_crawler: "Likely crawlers"page_view_embed: "Embedded pageviews"page_view_other: "Other traffic"yaxis: "Day"@@ -2980,6 +2981,7 @@ en:enable_horizon_high_context_topic_cards: "Shows richer topic cards in the Horizon theme, including topic excerpts, tags, and plugin extended information."granular_anonymous_and_logged_in_groups_permissions: "Makes group-based site setting permissions treat 'everyone' as 'all logged-in users', and enables the new 'anonymous_users' and 'logged_in_users' pseudogroups so admins can explicitly target anonymous visitors and authenticated users."dashboard_improvements: "Enables the redesigned admin dashboard that lets you pin custom reports and includes improved site traffic, engagement, and other analytics tied to community success and value."+ improved_crawler_detection: "Enables improved crawler tracking which can better detect likely crawlers and ensure traffic reporting is more accurate."enable_invite_modal_with_roles: "Enables a redesigned invite flow that lets admins invite someone directly as an admin. The invitee becomes a moderator on signup and an admin once the inviter confirms via email."update_pending_users_reminder_default: "We are updating the default for the {{setting:pending_users_reminder_delay_minutes}} setting to 30 minutes so that staff are promptly notified when a new member is waiting for approval. If you have previously changed this setting, this change will have no effect."diff --git a/config/site_settings.yml b/config/site_settings.ymlindex 56713fcc1a5..2a458143c97 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -653,9 +653,6 @@ basic:max: 86400hidden: trueclient: true- experimental_detect_crawler_pageviews:- default: false- hidden: truecrawler_automation_user_agents:default: "HeadlessChrome|Playwright|Puppeteer|Selenium|PhantomJS|jsdom"hidden: true@@ -4830,6 +4827,15 @@ experimental:status: "beta"impact: "feature,admins"learn_more_url: "https://meta.discourse.org/t/-/404528"+ improved_crawler_detection:+ default: false+ hidden: true+ client: true+ upcoming_change:+ status: "conceptual"+ impact: "other,all_members"+ allow_enabled_for:+ - everyoneadmin_dashboard_reports_seeded:default: falsehidden: truediff --git a/db/migrate/20260728033946_create_browser_pageview_crawler_daily_rollups.rb b/db/migrate/20260728033946_create_browser_pageview_crawler_daily_rollups.rbnew file mode 100644index 00000000000..37e5f42cc64--- /dev/null+++ b/db/migrate/20260728033946_create_browser_pageview_crawler_daily_rollups.rb@@ -0,0 +1,16 @@+# frozen_string_literal: true++class CreateBrowserPageviewCrawlerDailyRollups < ActiveRecord::Migration[8.0]+ def change+ create_table :browser_pageview_crawler_daily_rollups do |t|+ t.date :date, null: false+ t.boolean :logged_in, null: false+ t.bigint :count, null: false+ end++ add_index :browser_pageview_crawler_daily_rollups,+ %i[date logged_in],+ unique: true,+ name: "idx_bpcrawler_rollups_date_logged_in_unique"+ end+enddiff --git a/db/post_migrate/20260728050038_remove_experimental_detect_crawler_pageviews_setting.rb b/db/post_migrate/20260728050038_remove_experimental_detect_crawler_pageviews_setting.rbnew file mode 100644index 00000000000..36919853379--- /dev/null+++ b/db/post_migrate/20260728050038_remove_experimental_detect_crawler_pageviews_setting.rb@@ -0,0 +1,10 @@+# frozen_string_literal: true+class RemoveExperimentalDetectCrawlerPageviewsSetting < ActiveRecord::Migration[8.0]+ def up+ execute "DELETE FROM site_settings WHERE name = 'experimental_detect_crawler_pageviews'"+ end++ def down+ raise ActiveRecord::IrreversibleMigration+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex e6f2aa9982f..6459af85a3f 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -1978,6 +1978,37 @@ CREATE SEQUENCE public.browser_pageview_country_daily_rollups_id_seqALTER SEQUENCE public.browser_pageview_country_daily_rollups_id_seq OWNED BY public.browser_pageview_country_daily_rollups.id;+--+-- Name: browser_pageview_crawler_daily_rollups; Type: TABLE; Schema: public; Owner: -+--++CREATE TABLE public.browser_pageview_crawler_daily_rollups (+ id bigint NOT NULL,+ date date NOT NULL,+ logged_in boolean NOT NULL,+ count bigint NOT NULL+);+++--+-- Name: browser_pageview_crawler_daily_rollups_id_seq; Type: SEQUENCE; Schema: public; Owner: -+--++CREATE SEQUENCE public.browser_pageview_crawler_daily_rollups_id_seq+ START WITH 1+ INCREMENT BY 1+ NO MINVALUE+ NO MAXVALUE+ CACHE 1;+++--+-- Name: browser_pageview_crawler_daily_rollups_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -+--++ALTER SEQUENCE public.browser_pageview_crawler_daily_rollups_id_seq OWNED BY public.browser_pageview_crawler_daily_rollups.id;++---- Name: browser_pageview_event_scores; Type: TABLE; Schema: public; Owner: ---@@ -12760,6 +12791,13 @@ ALTER TABLE ONLY public.bookmarks ALTER COLUMN id SET DEFAULT nextval('public.boALTER TABLE ONLY public.browser_pageview_country_daily_rollups ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_country_daily_rollups_id_seq'::regclass);+--+-- Name: browser_pageview_crawler_daily_rollups id; Type: DEFAULT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.browser_pageview_crawler_daily_rollups ALTER COLUMN id SET DEFAULT nextval('public.browser_pageview_crawler_daily_rollups_id_seq'::regclass);++---- Name: browser_pageview_event_scores id; Type: DEFAULT; Schema: public; Owner: ---@@ -15053,6 +15091,14 @@ ALTER TABLE ONLY public.browser_pageview_country_daily_rollupsADD CONSTRAINT browser_pageview_country_daily_rollups_pkey PRIMARY KEY (id);+--+-- Name: browser_pageview_crawler_daily_rollups browser_pageview_crawler_daily_rollups_pkey; Type: CONSTRAINT; Schema: public; Owner: -+--++ALTER TABLE ONLY public.browser_pageview_crawler_daily_rollups+ ADD CONSTRAINT browser_pageview_crawler_daily_rollups_pkey PRIMARY KEY (id);++---- Name: browser_pageview_event_scores browser_pageview_event_scores_pkey; Type: CONSTRAINT; Schema: public; Owner: ---@@ -17529,6 +17575,13 @@ CREATE UNIQUE INDEX idx_bookmarks_user_polymorphic_unique ON public.bookmarks USCREATE UNIQUE INDEX idx_bpcd_rollups_date_country_unique ON public.browser_pageview_country_daily_rollups USING btree (date, country_code) NULLS NOT DISTINCT;+--+-- Name: idx_bpcrawler_rollups_date_logged_in_unique; Type: INDEX; Schema: public; Owner: -+--++CREATE UNIQUE INDEX idx_bpcrawler_rollups_date_logged_in_unique ON public.browser_pageview_crawler_daily_rollups USING btree (date, logged_in);++---- Name: idx_bpe_created_at_country_code; Type: INDEX; Schema: public; Owner: ---@@ -23082,7 +23135,9 @@ INSERT INTO "schema_migrations" (version) VALUES('20260728162516'),('20260728134532'),('20260728071552'),+('20260728050038'),('20260728045008'),+('20260728033946'),('20260727085824'),('20260727035337'),('20260723183001'),diff --git a/lib/crawler_scorer.rb b/lib/crawler_scorer.rbindex 05aa6507913..2f6c2044246 100644--- a/lib/crawler_scorer.rb+++ b/lib/crawler_scorer.rb@@ -1,7 +1,7 @@# frozen_string_literal: trueclass CrawlerScorer- BOT_SCORE_THRESHOLD = 60+ BOT_SCORE_THRESHOLD = 55AUTOMATION_UA_SCORE = 100diff --git a/spec/fabricators/browser_pageview_crawler_daily_rollup_fabricator.rb b/spec/fabricators/browser_pageview_crawler_daily_rollup_fabricator.rbnew file mode 100644index 00000000000..cfdf0a94054--- /dev/null+++ b/spec/fabricators/browser_pageview_crawler_daily_rollup_fabricator.rb@@ -0,0 +1,7 @@+# frozen_string_literal: true++Fabricator(:browser_pageview_crawler_daily_rollup) do+ date { Date.current }+ logged_in false+ count 1+enddiff --git a/spec/jobs/detect_crawler_pageviews_spec.rb b/spec/jobs/detect_crawler_pageviews_spec.rbindex dde737a08ee..40abf4e3401 100644--- a/spec/jobs/detect_crawler_pageviews_spec.rb+++ b/spec/jobs/detect_crawler_pageviews_spec.rb@@ -1,15 +1,15 @@# frozen_string_literal: trueRSpec.describe Jobs::DetectCrawlerPageviews do- it "does nothing when detection is disabled" do- SiteSetting.experimental_detect_crawler_pageviews = false+ it "does nothing when the improved crawler detection change is disabled" do+ SiteSetting.improved_crawler_detection = falseCrawlerScorer.expects(:score!).neverdescribed_class.new.execute({})endit "scores a one hour wide window, held back by the beacon settle period" do- SiteSetting.experimental_detect_crawler_pageviews = true+ SiteSetting.improved_crawler_detection = truefreeze_timeCrawlerScorer.expects(:score!).with(window_start: 90.minutes.ago, window_end: 30.minutes.ago)diff --git a/spec/jobs/maintain_browser_pageview_rollups_spec.rb b/spec/jobs/maintain_browser_pageview_rollups_spec.rbindex b62f225cdaa..a86fe5fafcd 100644--- a/spec/jobs/maintain_browser_pageview_rollups_spec.rb+++ b/spec/jobs/maintain_browser_pageview_rollups_spec.rb@@ -110,6 +110,70 @@ RSpec.describe Jobs::MaintainBrowserPageviewRollups doendend+ context "when aggregating crawler rollups" do+ let(:above_threshold) { CrawlerScorer::BOT_SCORE_THRESHOLD + 1 }++ before { freeze_time(Time.utc(2026, 6, 20, 12, 0, 0)) }++ it "does nothing when improved_crawler_detection is disabled" do+ Fabricate(+ :browser_pageview_event,+ score: above_threshold,+ created_at: Time.utc(2026, 6, 20, 9),+ )++ job.execute({})++ expect(BrowserPageviewCrawlerDailyRollup.count).to eq(0)+ end++ context "when improved_crawler_detection is enabled" do+ before { SiteSetting.improved_crawler_detection = true }++ it "backfills the full event history on the first run" do+ Fabricate(+ :browser_pageview_event,+ score: above_threshold,+ created_at: Time.utc(2026, 5, 1, 9),+ )+ Fabricate(+ :browser_pageview_event,+ score: above_threshold,+ created_at: Time.utc(2026, 6, 20, 9),+ )++ job.execute({})++ expect(BrowserPageviewCrawlerDailyRollup.sum(:count)).to eq(2)+ end++ it "only refreshes yesterday and today once history has been rolled up" do+ Fabricate(+ :browser_pageview_event,+ score: above_threshold,+ created_at: Time.utc(2026, 6, 20, 9),+ )+ job.execute({})++ Fabricate(+ :browser_pageview_event,+ score: above_threshold,+ created_at: Time.utc(2026, 5, 1, 9),+ )+ Fabricate(+ :browser_pageview_event,+ score: above_threshold,+ created_at: Time.utc(2026, 6, 19, 9),+ )+ job.execute({})++ expect(+ BrowserPageviewCrawlerDailyRollup.order(:date).pluck(:date, :count),+ ).to contain_exactly([Date.new(2026, 6, 19), 1], [Date.new(2026, 6, 20), 1])+ end+ end+ end+context "when aggregating engagement rollups" dobefore { freeze_time(Time.utc(2026, 6, 20, 12, 0, 0)) }diff --git a/spec/models/browser_pageview_crawler_daily_rollup_spec.rb b/spec/models/browser_pageview_crawler_daily_rollup_spec.rbnew file mode 100644index 00000000000..755348f5668--- /dev/null+++ b/spec/models/browser_pageview_crawler_daily_rollup_spec.rb@@ -0,0 +1,69 @@+# frozen_string_literal: true++RSpec.describe BrowserPageviewCrawlerDailyRollup do+ describe ".aggregate" do+ let(:start_date) { 3.days.ago.to_date }+ let(:end_date) { Date.current }+ let(:above_threshold) { CrawlerScorer::BOT_SCORE_THRESHOLD + 1 }++ it "groups likely crawler events by date and logged in state" do+ user = Fabricate(:user)+ yesterday = 1.day.ago+ today = Time.current++ Fabricate(:browser_pageview_event, score: above_threshold, created_at: yesterday)+ Fabricate(:browser_pageview_event, score: above_threshold, created_at: today)+ Fabricate(+ :browser_pageview_event,+ score: above_threshold,+ user_id: user.id,+ created_at: today,+ )++ described_class.aggregate(start_date: start_date, end_date: end_date)++ expect(described_class.order(:date, :logged_in).pluck(:date, :logged_in, :count)).to eq(+ [[yesterday.to_date, false, 1], [today.to_date, false, 1], [today.to_date, true, 1]],+ )+ end++ it "ignores events at or below the threshold, and unscored events" do+ Fabricate(:browser_pageview_event, score: CrawlerScorer::BOT_SCORE_THRESHOLD)+ Fabricate(:browser_pageview_event, score: 0)+ Fabricate(:browser_pageview_event, score: nil)+ Fabricate(:browser_pageview_event, score: above_threshold)++ described_class.aggregate(start_date: start_date, end_date: end_date)++ expect(described_class.sum(:count)).to eq(1)+ end++ it "only aggregates events within the requested date range" do+ Fabricate(:browser_pageview_event, score: above_threshold, created_at: 10.days.ago)+ Fabricate(:browser_pageview_event, score: above_threshold, created_at: 1.day.ago)++ described_class.aggregate(start_date: start_date, end_date: end_date)++ expect(described_class.sum(:count)).to eq(1)+ end++ it "is idempotent and refreshes counts when re-aggregating" do+ Fabricate(:browser_pageview_event, score: above_threshold)+ described_class.aggregate(start_date: start_date, end_date: end_date)++ Fabricate(:browser_pageview_event, score: above_threshold)+ described_class.aggregate(start_date: start_date, end_date: end_date)++ expect(described_class.count).to eq(1)+ expect(described_class.sum(:count)).to eq(2)+ end++ it "is a no-op when no likely crawler events exist in the range" do+ Fabricate(:browser_pageview_event, score: nil)++ expect {+ described_class.aggregate(start_date: start_date, end_date: end_date)+ }.not_to change { described_class.count }+ end+ end+enddiff --git a/spec/services/admin_dashboard_site_traffic_spec.rb b/spec/services/admin_dashboard_site_traffic_spec.rbindex 3aec552db7f..dfd0ca8892b 100644--- a/spec/services/admin_dashboard_site_traffic_spec.rb+++ b/spec/services/admin_dashboard_site_traffic_spec.rb@@ -34,6 +34,7 @@ RSpec.describe AdminDashboardSiteTraffic doanonymous: "page_view_anon_browser",embedded: "page_view_embed",crawlers: "page_view_crawler",+ likely_crawlers: "page_view_likely_crawler",}.fetch(id)end@@ -1085,5 +1086,131 @@ RSpec.describe AdminDashboardSiteTraffic do)endend++ context "for likely crawlers" do+ before { SiteSetting.improved_crawler_detection = true }++ it "orders likely crawlers ahead of known crawlers" do+ SiteSetting.embed_topics_list = true+ Fabricate(:embeddable_host)++ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")++ expect(response[:pageview_series].map { |series| series[:req] }).to eq(+ %w[+ page_view_logged_in_browser+ page_view_anon_browser+ page_view_embed+ page_view_likely_crawler+ page_view_crawler+ ],+ )+ end++ it "reclassifies likely crawlers out of the logged in and anonymous series" do+ Fabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 10)+ Fabricate(:anonymous_browser_application_request, date: "2026-05-01", count: 20)++ Fabricate(+ :browser_pageview_crawler_daily_rollup,+ date: Date.new(2026, 5, 1),+ logged_in: true,+ count: 4,+ )+ Fabricate(+ :browser_pageview_crawler_daily_rollup,+ date: Date.new(2026, 5, 1),+ logged_in: false,+ count: 15,+ )++ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")++ expect(traffic_series_data(response, :logged_in)).to eq([traffic_point("2026-05-01", 6)])+ expect(traffic_series_data(response, :anonymous)).to eq([traffic_point("2026-05-01", 5)])+ expect(traffic_series_data(response, :likely_crawlers)).to eq(+ [traffic_point("2026-05-01", 19)],+ )+ expect(response[:kpis][:browser_pageviews][:value]).to eq(11)+ end++ it "never drives a series below zero when the rollup exceeds recorded pageviews" do+ Fabricate(:anonymous_browser_application_request, date: "2026-05-01", count: 2)+ Fabricate(+ :browser_pageview_crawler_daily_rollup,+ date: Date.new(2026, 5, 1),+ logged_in: false,+ count: 9,+ )++ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")++ expect(traffic_series_data(response, :anonymous)).to eq([traffic_point("2026-05-01", 0)])+ expect(traffic_series_data(response, :likely_crawlers)).to eq(+ [traffic_point("2026-05-01", 9)],+ )+ end++ it "ignores rollups outside the requested range" do+ Fabricate(:anonymous_browser_application_request, date: "2026-05-01", count: 10)+ Fabricate(+ :browser_pageview_crawler_daily_rollup,+ date: Date.new(2026, 4, 20),+ logged_in: false,+ count: 7,+ )++ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")++ expect(traffic_series_data(response, :anonymous)).to eq([traffic_point("2026-05-01", 10)])+ expect(traffic_series_data(response, :likely_crawlers)).to eq(+ [traffic_point("2026-05-01", 0)],+ )+ end++ it "excludes anonymous crawlers when login is required" do+ SiteSetting.login_required = true++ Fabricate(:logged_in_browser_application_request, date: "2026-05-01", count: 10)+ Fabricate(+ :browser_pageview_crawler_daily_rollup,+ date: Date.new(2026, 5, 1),+ logged_in: true,+ count: 3,+ )+ Fabricate(+ :browser_pageview_crawler_daily_rollup,+ date: Date.new(2026, 5, 1),+ logged_in: false,+ count: 8,+ )++ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")++ expect(traffic_series_data(response, :logged_in)).to eq([traffic_point("2026-05-01", 7)])+ expect(traffic_series_data(response, :likely_crawlers)).to eq(+ [traffic_point("2026-05-01", 3)],+ )+ end++ it "leaves the series out and keeps counts intact when the change is disabled" do+ SiteSetting.improved_crawler_detection = false++ Fabricate(:anonymous_browser_application_request, date: "2026-05-01", count: 10)+ Fabricate(+ :browser_pageview_crawler_daily_rollup,+ date: Date.new(2026, 5, 1),+ logged_in: false,+ count: 6,+ )++ response = build_traffic(start_date: "2026-05-01", end_date: "2026-05-01")++ expect(traffic_series_data(response, :anonymous)).to eq([traffic_point("2026-05-01", 10)])+ expect(response[:pageview_series].map { |series| series[:req] }).not_to include(+ "page_view_likely_crawler",+ )+ end+ endendend