A platform for community discussion. Free, open, simple.
FIX: Show likely crawlers in the site traffic report (#42292)
Krzysztof Kotlarek170da10d7455
diff --git a/app/models/concerns/reports/site_traffic.rb b/app/models/concerns/reports/site_traffic.rbindex 69f3ebe747e..da003046c8d 100644--- a/app/models/concerns/reports/site_traffic.rb+++ b/app/models/concerns/reports/site_traffic.rb@@ -26,31 +26,55 @@ module Reports::SiteTrafficpage_view_anon_browser: ApplicationRequest.req_types[:page_view_anon_browser],).first+ likely_crawlers_enabled = UpcomingChanges.enabled?(:improved_crawler_detection)+data =DB.query(<<~SQL,+ WITH likely_crawlers AS (+ SELECT+ date,+ COALESCE(SUM(count) FILTER (WHERE logged_in), 0)::bigint AS logged_in,+ COALESCE(SUM(count) FILTER (WHERE NOT logged_in), 0)::bigint AS anonymous+ FROM browser_pageview_crawler_daily_rollups+ WHERE :likely_crawlers_enabled+ AND date >= :start_date+ AND date <= :end_date+ GROUP BY date+ )SELECT- date,- SUM(CASE WHEN req_type = :page_view_logged_in_browser THEN count ELSE 0 END) AS page_view_logged_in_browser,- SUM(CASE WHEN req_type = :page_view_anon_browser THEN count ELSE 0 END) AS page_view_anon_browser,- SUM(CASE WHEN req_type = :page_view_crawler THEN count ELSE 0 END) AS page_view_crawler,- SUM(CASE WHEN req_type = :page_view_embed THEN count ELSE 0 END) AS page_view_embed,+ ar.date,+ GREATEST(+ 0,+ SUM(CASE WHEN ar.req_type = :page_view_logged_in_browser THEN ar.count ELSE 0 END)+ - COALESCE(MAX(lc.logged_in), 0)+ ) AS page_view_logged_in_browser,+ GREATEST(+ 0,+ SUM(CASE WHEN ar.req_type = :page_view_anon_browser THEN ar.count ELSE 0 END)+ - COALESCE(MAX(lc.anonymous), 0)+ ) AS page_view_anon_browser,+ (COALESCE(MAX(lc.logged_in), 0) + COALESCE(MAX(lc.anonymous), 0)) AS page_view_likely_crawler,+ SUM(CASE WHEN ar.req_type = :page_view_crawler THEN ar.count ELSE 0 END) AS page_view_crawler,+ SUM(CASE WHEN ar.req_type = :page_view_embed THEN ar.count ELSE 0 END) AS page_view_embed,SUM(- CASE WHEN req_type = :page_view_anon THEN count- WHEN req_type = :page_view_logged_in THEN count- WHEN req_type = :page_view_anon_browser THEN -count- WHEN req_type = :page_view_logged_in_browser THEN -count+ CASE WHEN ar.req_type = :page_view_anon THEN ar.count+ WHEN ar.req_type = :page_view_logged_in THEN ar.count+ WHEN ar.req_type = :page_view_anon_browser THEN -ar.count+ WHEN ar.req_type = :page_view_logged_in_browser THEN -ar.countELSE 0END) AS page_view_other- FROM application_requests- WHERE date >= :start_date AND date <= :end_date AND date >= :first_browser_pageview_date+ FROM application_requests ar+ LEFT JOIN likely_crawlers lc ON lc.date = ar.date+ WHERE ar.date >= :start_date AND ar.date <= :end_date AND ar.date >= :first_browser_pageview_date- GROUP BY date- ORDER BY date ASC+ GROUP BY ar.date+ ORDER BY ar.date ASCSQLstart_date: report.start_date,end_date: report.end_date,+ likely_crawlers_enabled: likely_crawlers_enabled,page_view_anon: ApplicationRequest.req_types[:page_view_anon],page_view_crawler: ApplicationRequest.req_types[:page_view_crawler],page_view_logged_in: ApplicationRequest.req_types[:page_view_logged_in],@@ -73,14 +97,24 @@ module Reports::SiteTrafficcolor: SERIES_COLORS.fetch("page_view_anon_browser"),data: data.map { |row| { x: row.date, y: row.page_view_anon_browser } },},- {- req: "page_view_crawler",- label: I18n.t("reports.site_traffic.xaxis.page_view_crawler"),- color: SERIES_COLORS.fetch("page_view_crawler"),- data: data.map { |row| { x: row.date, y: row.page_view_crawler } },- },]+ if likely_crawlers_enabled+ report.data << {+ req: "page_view_likely_crawler",+ label: I18n.t("reports.site_traffic.xaxis.page_view_likely_crawler"),+ color: SERIES_COLORS.fetch("page_view_likely_crawler"),+ data: data.map { |row| { x: row.date, y: row.page_view_likely_crawler } },+ }+ end++ report.data << {+ req: "page_view_crawler",+ label: I18n.t("reports.site_traffic.xaxis.page_view_crawler"),+ color: SERIES_COLORS.fetch("page_view_crawler"),+ data: data.map { |row| { x: row.date, y: row.page_view_crawler } },+ }+if EmbeddableHost.exists?report.data << {req: "page_view_embed",diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rbindex 8ad1b81ede4..9ffd3362999 100644--- a/spec/models/report_spec.rb+++ b/spec/models/report_spec.rb@@ -1726,6 +1726,63 @@ RSpec.describe Report doexpect(reports.data.map { |r| r[:req] }).not_to include("page_view_embed")end++ context "for likely crawlers" do+ before { SiteSetting.improved_crawler_detection = true }++ it "reclassifies likely crawlers out of the logged in and anonymous series" do+ 10.times { ApplicationRequest.increment!(:page_view_logged_in_browser) }+ 20.times { ApplicationRequest.increment!(:page_view_anon_browser) }+ CachedCounting.flush++ Fabricate(+ :browser_pageview_crawler_daily_rollup,+ date: Time.zone.today,+ logged_in: true,+ count: 4,+ )+ Fabricate(+ :browser_pageview_crawler_daily_rollup,+ date: Time.zone.today,+ logged_in: false,+ count: 6,+ )++ series = reports.data.index_by { |r| r[:req] }++ expect(series.keys).to eq(+ %w[+ page_view_logged_in_browser+ page_view_anon_browser+ page_view_likely_crawler+ page_view_crawler+ page_view_other+ ],+ )+ expect(series["page_view_logged_in_browser"][:data][0][:y]).to eq(6)+ expect(series["page_view_anon_browser"][:data][0][:y]).to eq(14)+ expect(series["page_view_likely_crawler"][:data][0][:y]).to eq(10)+ end++ it "leaves the series out and keeps counts intact when the change is disabled" do+ SiteSetting.improved_crawler_detection = false++ 10.times { ApplicationRequest.increment!(:page_view_anon_browser) }+ CachedCounting.flush++ Fabricate(+ :browser_pageview_crawler_daily_rollup,+ date: Time.zone.today,+ logged_in: false,+ count: 6,+ )++ series = reports.data.index_by { |r| r[:req] }++ expect(series.keys).not_to include("page_view_likely_crawler")+ expect(series["page_view_anon_browser"][:data][0][:y]).to eq(10)+ end+ endendend