Skip to content

A platform for community discussion. Free, open, simple.

FIX: Show likely crawlers in the site traffic report (#42292)

Krzysztof Kotlarek170da10d7455
diff --git a/app/models/concerns/reports/site_traffic.rb b/app/models/concerns/reports/site_traffic.rb
index 69f3ebe747e..da003046c8d 100644
--- a/app/models/concerns/reports/site_traffic.rb
+++ b/app/models/concerns/reports/site_traffic.rb
@@ -26,31 +26,55 @@ module Reports::SiteTraffic
page_view_anon_browser: ApplicationRequest.req_types[:page_view_anon_browser],
).first
+ likely_crawlers_enabled = UpcomingChanges.enabled?(:improved_crawler_detection)
+
data =
DB.query(
<<~SQL,
+ WITH likely_crawlers AS (
+ SELECT
+ date,
+ COALESCE(SUM(count) FILTER (WHERE logged_in), 0)::bigint AS logged_in,
+ COALESCE(SUM(count) FILTER (WHERE NOT logged_in), 0)::bigint AS anonymous
+ FROM browser_pageview_crawler_daily_rollups
+ WHERE :likely_crawlers_enabled
+ AND date >= :start_date
+ AND date <= :end_date
+ GROUP BY date
+ )
SELECT
- date,
- SUM(CASE WHEN req_type = :page_view_logged_in_browser THEN count ELSE 0 END) AS page_view_logged_in_browser,
- SUM(CASE WHEN req_type = :page_view_anon_browser THEN count ELSE 0 END) AS page_view_anon_browser,
- SUM(CASE WHEN req_type = :page_view_crawler THEN count ELSE 0 END) AS page_view_crawler,
- SUM(CASE WHEN req_type = :page_view_embed THEN count ELSE 0 END) AS page_view_embed,
+ ar.date,
+ GREATEST(
+ 0,
+ SUM(CASE WHEN ar.req_type = :page_view_logged_in_browser THEN ar.count ELSE 0 END)
+ - COALESCE(MAX(lc.logged_in), 0)
+ ) AS page_view_logged_in_browser,
+ GREATEST(
+ 0,
+ SUM(CASE WHEN ar.req_type = :page_view_anon_browser THEN ar.count ELSE 0 END)
+ - COALESCE(MAX(lc.anonymous), 0)
+ ) AS page_view_anon_browser,
+ (COALESCE(MAX(lc.logged_in), 0) + COALESCE(MAX(lc.anonymous), 0)) AS page_view_likely_crawler,
+ SUM(CASE WHEN ar.req_type = :page_view_crawler THEN ar.count ELSE 0 END) AS page_view_crawler,
+ SUM(CASE WHEN ar.req_type = :page_view_embed THEN ar.count ELSE 0 END) AS page_view_embed,
SUM(
- CASE WHEN req_type = :page_view_anon THEN count
- WHEN req_type = :page_view_logged_in THEN count
- WHEN req_type = :page_view_anon_browser THEN -count
- WHEN req_type = :page_view_logged_in_browser THEN -count
+ CASE WHEN ar.req_type = :page_view_anon THEN ar.count
+ WHEN ar.req_type = :page_view_logged_in THEN ar.count
+ WHEN ar.req_type = :page_view_anon_browser THEN -ar.count
+ WHEN ar.req_type = :page_view_logged_in_browser THEN -ar.count
ELSE 0
END
) AS page_view_other
- FROM application_requests
- WHERE date >= :start_date AND date <= :end_date AND date >= :first_browser_pageview_date
+ FROM application_requests ar
+ LEFT JOIN likely_crawlers lc ON lc.date = ar.date
+ WHERE ar.date >= :start_date AND ar.date <= :end_date AND ar.date >= :first_browser_pageview_date
- GROUP BY date
- ORDER BY date ASC
+ GROUP BY ar.date
+ ORDER BY ar.date ASC
SQL
start_date: report.start_date,
end_date: report.end_date,
+ likely_crawlers_enabled: likely_crawlers_enabled,
page_view_anon: ApplicationRequest.req_types[:page_view_anon],
page_view_crawler: ApplicationRequest.req_types[:page_view_crawler],
page_view_logged_in: ApplicationRequest.req_types[:page_view_logged_in],
@@ -73,14 +97,24 @@ module Reports::SiteTraffic
color: SERIES_COLORS.fetch("page_view_anon_browser"),
data: data.map { |row| { x: row.date, y: row.page_view_anon_browser } },
},
- {
- req: "page_view_crawler",
- label: I18n.t("reports.site_traffic.xaxis.page_view_crawler"),
- color: SERIES_COLORS.fetch("page_view_crawler"),
- data: data.map { |row| { x: row.date, y: row.page_view_crawler } },
- },
]
+ if likely_crawlers_enabled
+ report.data << {
+ req: "page_view_likely_crawler",
+ label: I18n.t("reports.site_traffic.xaxis.page_view_likely_crawler"),
+ color: SERIES_COLORS.fetch("page_view_likely_crawler"),
+ data: data.map { |row| { x: row.date, y: row.page_view_likely_crawler } },
+ }
+ end
+
+ report.data << {
+ req: "page_view_crawler",
+ label: I18n.t("reports.site_traffic.xaxis.page_view_crawler"),
+ color: SERIES_COLORS.fetch("page_view_crawler"),
+ data: data.map { |row| { x: row.date, y: row.page_view_crawler } },
+ }
+
if EmbeddableHost.exists?
report.data << {
req: "page_view_embed",
diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rb
index 8ad1b81ede4..9ffd3362999 100644
--- a/spec/models/report_spec.rb
+++ b/spec/models/report_spec.rb
@@ -1726,6 +1726,63 @@ RSpec.describe Report do
expect(reports.data.map { |r| r[:req] }).not_to include("page_view_embed")
end
+
+ context "for likely crawlers" do
+ before { SiteSetting.improved_crawler_detection = true }
+
+ it "reclassifies likely crawlers out of the logged in and anonymous series" do
+ 10.times { ApplicationRequest.increment!(:page_view_logged_in_browser) }
+ 20.times { ApplicationRequest.increment!(:page_view_anon_browser) }
+ CachedCounting.flush
+
+ Fabricate(
+ :browser_pageview_crawler_daily_rollup,
+ date: Time.zone.today,
+ logged_in: true,
+ count: 4,
+ )
+ Fabricate(
+ :browser_pageview_crawler_daily_rollup,
+ date: Time.zone.today,
+ logged_in: false,
+ count: 6,
+ )
+
+ series = reports.data.index_by { |r| r[:req] }
+
+ expect(series.keys).to eq(
+ %w[
+ page_view_logged_in_browser
+ page_view_anon_browser
+ page_view_likely_crawler
+ page_view_crawler
+ page_view_other
+ ],
+ )
+ expect(series["page_view_logged_in_browser"][:data][0][:y]).to eq(6)
+ expect(series["page_view_anon_browser"][:data][0][:y]).to eq(14)
+ expect(series["page_view_likely_crawler"][:data][0][:y]).to eq(10)
+ end
+
+ it "leaves the series out and keeps counts intact when the change is disabled" do
+ SiteSetting.improved_crawler_detection = false
+
+ 10.times { ApplicationRequest.increment!(:page_view_anon_browser) }
+ CachedCounting.flush
+
+ Fabricate(
+ :browser_pageview_crawler_daily_rollup,
+ date: Time.zone.today,
+ logged_in: false,
+ count: 6,
+ )
+
+ series = reports.data.index_by { |r| r[:req] }
+
+ expect(series.keys).not_to include("page_view_likely_crawler")
+ expect(series["page_view_anon_browser"][:data][0][:y]).to eq(10)
+ end
+ end
end
end