A platform for community discussion. Free, open, simple.
PERF: Improve activity by category report by doing it one-pass across the (e.g.) 2 year date period (#40945)
Natalie Taye6a583fa12bb
diff --git a/app/models/concerns/reports/activity_by_category.rb b/app/models/concerns/reports/activity_by_category.rbindex c5d0737e556..95e06c9fda1 100644--- a/app/models/concerns/reports/activity_by_category.rb+++ b/app/models/concerns/reports/activity_by_category.rb@@ -56,11 +56,14 @@ module Reports::ActivityByCategorysecure_category_ids =report.current_user&.admin? ? nil : Guardian.new(report.current_user).secure_category_ids- current_period =- period_data(report.start_date, report.end_date, requested_ids, secure_category_ids)-- prior_start = report.start_date - (report.end_date - report.start_date)- prior_period = period_data(prior_start, report.start_date, requested_ids, secure_category_ids)+ current_period, prior_period =+ period_data(+ report.prev_start_date,+ report.start_date,+ report.end_date,+ requested_ids,+ secure_category_ids,+ )total_current =current_period.values.sum { |row| row[:topics] + row[:posts] + row[:page_views] }@@ -106,32 +109,39 @@ module Reports::ActivityByCategory"#{sign}#{change}%"end- def period_data(period_start, period_end, requested_ids, secure_category_ids)+ def period_data(prev_start, current_start, current_end, requested_ids, secure_category_ids)builder = DB.build <<~SQLSELECTc.id,c.name,c.color,c.slug,- COALESCE(t.topics, 0) AS topics,- COALESCE(p.posts, 0) AS posts,- COALESCE(v.page_views, 0) AS page_views+ COALESCE(t.topics_current, 0) AS topics_current,+ COALESCE(p.posts_current, 0) AS posts_current,+ COALESCE(v.page_views_current, 0) AS page_views_current,+ COALESCE(t.topics_prior, 0) AS topics_prior,+ COALESCE(p.posts_prior, 0) AS posts_prior,+ COALESCE(v.page_views_prior, 0) AS page_views_priorFROM categories cLEFT JOIN (- SELECT category_id, COUNT(*) AS topics+ SELECT category_id,+ COUNT(*) FILTER (WHERE created_at >= :current_start) AS topics_current,+ COUNT(*) FILTER (WHERE created_at <= :current_start) AS topics_priorFROM topics- WHERE created_at >= :period_start- AND created_at <= :period_end+ WHERE created_at >= :prev_start+ AND created_at <= :current_endAND deleted_at IS NULLAND archetype = 'regular'GROUP BY category_id) t ON t.category_id = c.idLEFT JOIN (- SELECT topics.category_id, COUNT(*) AS posts+ SELECT topics.category_id,+ COUNT(*) FILTER (WHERE posts.created_at >= :current_start) AS posts_current,+ COUNT(*) FILTER (WHERE posts.created_at <= :current_start) AS posts_priorFROM postsINNER JOIN topics ON topics.id = posts.topic_id- WHERE posts.created_at >= :period_start- AND posts.created_at <= :period_end+ WHERE posts.created_at >= :prev_start+ AND posts.created_at <= :current_endAND posts.deleted_at IS NULLAND posts.post_type = :regular_post_typeAND topics.deleted_at IS NULL@@ -140,11 +150,14 @@ module Reports::ActivityByCategory) p ON p.category_id = c.idLEFT JOIN (SELECT topics.category_id,- COALESCE(SUM(tvs.anonymous_views + tvs.logged_in_views), 0) AS page_views+ COALESCE(SUM(tvs.anonymous_views + tvs.logged_in_views)+ FILTER (WHERE tvs.viewed_at >= :current_start), 0) AS page_views_current,+ COALESCE(SUM(tvs.anonymous_views + tvs.logged_in_views)+ FILTER (WHERE tvs.viewed_at <= :current_start), 0) AS page_views_priorFROM topic_view_stats tvsINNER JOIN topics ON topics.id = tvs.topic_id- WHERE tvs.viewed_at >= :period_start- AND tvs.viewed_at <= :period_end+ WHERE tvs.viewed_at >= :prev_start+ AND tvs.viewed_at <= :current_endAND topics.deleted_at IS NULLAND topics.archetype = 'regular'GROUP BY topics.category_id@@ -152,9 +165,10 @@ module Reports::ActivityByCategory/*where*/SQL- builder.where(- "(COALESCE(t.topics, 0) + COALESCE(p.posts, 0) + COALESCE(v.page_views, 0)) > 0",- )+ builder.where(<<~SQL)+ (COALESCE(t.topics_current, 0) + COALESCE(p.posts_current, 0) + COALESCE(v.page_views_current, 0)) > 0+ OR (COALESCE(t.topics_prior, 0) + COALESCE(p.posts_prior, 0) + COALESCE(v.page_views_prior, 0)) > 0+ SQLif requested_ids.present?builder.where("c.id IN (:requested_ids)", requested_ids: requested_ids)@@ -162,24 +176,36 @@ module Reports::ActivityByCategorybuilder.secure_category(secure_category_ids) unless secure_category_ids.nil?- result = {}+ current_period = {}+ prior_period = {}builder.query(- period_start: period_start,- period_end: period_end,+ prev_start: prev_start,+ current_start: current_start,+ current_end: current_end,regular_post_type: Post.types[:regular],).each do |row|- result[row.id] = {- name: row.name,- color: row.color,- slug: row.slug,- topics: row.topics,- posts: row.posts,- page_views: row.page_views,- }+ if row.topics_current + row.posts_current + row.page_views_current > 0+ current_period[row.id] = {+ name: row.name,+ color: row.color,+ slug: row.slug,+ topics: row.topics_current,+ posts: row.posts_current,+ page_views: row.page_views_current,+ }+ end++ if row.topics_prior + row.posts_prior + row.page_views_prior > 0+ prior_period[row.id] = {+ topics: row.topics_prior,+ posts: row.posts_prior,+ page_views: row.page_views_prior,+ }+ endend- result+ [current_period, prior_period]endendenddiff --git a/spec/models/concerns/reports/activity_by_category_spec.rb b/spec/models/concerns/reports/activity_by_category_spec.rbindex 9472743bac2..891b33a08e6 100644--- a/spec/models/concerns/reports/activity_by_category_spec.rb+++ b/spec/models/concerns/reports/activity_by_category_spec.rb@@ -96,6 +96,22 @@ describe Reports::ActivityByCategory doexpect(formatted).to end_with("%")end+ it "computes share_change against the prior period" do+ grew = Fabricate(:category)+ shrank = Fabricate(:category)+ # current period activity+ 3.times { Fabricate(:topic, category: grew, created_at: start_date + 1.day) }+ Fabricate(:topic, category: shrank, created_at: start_date + 1.day)+ # prior period activity (start_date - 1.day falls in [prior_start, start_date])+ 3.times { Fabricate(:topic, category: shrank, created_at: start_date - 1.day) }++ report = build(filters: { category_ids: [grew.id, shrank.id] })++ expect(row(report, grew.id)[:share_change]).to be > 0+ expect(row(report, shrank.id)[:share_change]).to be < 0+ expect(row(report, shrank.id)[:share_change_formatted]).not_to start_with("+")+ end+it "returns empty rows when an explicit filter resolves to no valid ids" doFabricate(:category) # noise data — should NOT be returned via top-N fallbackFabricate(:topic, created_at: start_date + 1.day)