A platform for community discussion. Free, open, simple.
PERF: Warm the dashboard report cache for the default period (#42390)
Krzysztof Kotlarek183ef04c1ddc
diff --git a/app/jobs/scheduled/warm_dashboard_reports.rb b/app/jobs/scheduled/warm_dashboard_reports.rbnew file mode 100644index 00000000000..7a655185e87--- /dev/null+++ b/app/jobs/scheduled/warm_dashboard_reports.rb@@ -0,0 +1,22 @@+# frozen_string_literal: true++module Jobs+ class WarmDashboardReports < ::Jobs::Scheduled+ every 30.minutes++ STAFF_ACTIVITY_WINDOW = 7.days++ def execute(_args)+ return if !UpcomingChanges.enabled?(:dashboard_improvements)+ return if !recently_active_staff?++ AdminDashboardCacheWarmer.call+ end++ private++ def recently_active_staff?+ User.human_users.staff.where("last_seen_at > ?", STAFF_ACTIVITY_WINDOW.ago).exists?+ end+ end+enddiff --git a/app/models/report.rb b/app/models/report.rbindex 4c58f4cea68..8c18e1a2c89 100644--- a/app/models/report.rb+++ b/app/models/report.rb@@ -366,7 +366,7 @@ class Reportenddef self.cache(report)- duration = report.error == :exception ? 1.minute : 35.minutes+ duration = report.error == :exception ? 1.minute : 60.minutesDiscourse.cache.write(cache_key(report), report.as_json, expires_in: duration)enddiff --git a/app/services/admin_dashboard_cache_warmer.rb b/app/services/admin_dashboard_cache_warmer.rbnew file mode 100644index 00000000000..3ba4ae0feca--- /dev/null+++ b/app/services/admin_dashboard_cache_warmer.rb@@ -0,0 +1,39 @@+# frozen_string_literal: true++class AdminDashboardCacheWarmer+ WINDOW_DAYS = 30+ # The dashboard sends dates computed in the viewer's browser timezone, which can+ # sit a day either side of the server's.+ DATE_OFFSETS = (-1..1)++ def self.call+ windows.each do |window|+ report_specs.each do |spec|+ report = Report.find(spec[:type], spec[:opts].merge(window))+ Report.cache(report) if report && report.error.blank?+ end+ end+ end++ def self.report_specs+ kpi_reports =+ (+ AdminDashboardHighlights.enabled_kpis.map { |kpi| kpi[:report] } ++ AdminDashboardEngagement::KPI_REPORTS.values+ ).uniq++ kpi_reports.map { |type| { type: type, opts: { facets: %i[prev_period] } } } << {+ type: "trust_level_pipeline",+ opts: {+ },+ }+ end++ def self.windows+ DATE_OFFSETS.map do |offset|+ end_date = Time.zone.now.to_date + offset++ { start_date: (end_date - (WINDOW_DAYS - 1)).beginning_of_day, end_date: end_date.end_of_day }+ end+ end+enddiff --git a/app/services/admin_dashboard_highlights.rb b/app/services/admin_dashboard_highlights.rbindex e1f6299e1d7..a60d2507950 100644--- a/app/services/admin_dashboard_highlights.rb+++ b/app/services/admin_dashboard_highlights.rb@@ -15,6 +15,14 @@ class AdminDashboardHighlightsnew(start_date: start_date, end_date: end_date).buildend+ def self.enabled_kpis+ core_kpis = KPI_REPORTS.map { |type, report| { type: type, report: report } }++ (core_kpis + DiscoursePluginRegistry.admin_dashboard_highlight_kpis).reject do |kpi|+ kpi[:enabled].respond_to?(:call) && !kpi[:enabled].call+ end+ end+def initialize(start_date:, end_date:)@start_date = parse_date(start_date) || DEFAULT_RANGE_DAYS.days.ago.beginning_of_day@end_date = parse_date(end_date)&.end_of_day || Time.zone.now.end_of_day@@ -36,12 +44,6 @@ class AdminDashboardHighlightsenddef build_kpis- core_kpis = KPI_REPORTS.map { |type, report| { type: type, report: report } }- all_kpis = core_kpis + DiscoursePluginRegistry.admin_dashboard_highlight_kpis-- all_kpis.filter_map do |kpi|- next if kpi[:enabled].respond_to?(:call) && !kpi[:enabled].call- build_kpi(kpi[:type], kpi[:report])- end+ self.class.enabled_kpis.filter_map { |kpi| build_kpi(kpi[:type], kpi[:report]) }endenddiff --git a/app/services/admin_dashboard_site_traffic.rb b/app/services/admin_dashboard_site_traffic.rbindex 1f9eab972ef..30ce57cb9a3 100644--- a/app/services/admin_dashboard_site_traffic.rb+++ b/app/services/admin_dashboard_site_traffic.rb@@ -76,7 +76,7 @@ class AdminDashboardSiteTrafficreturn { rows: [], error: "exception" } if report.nil?# Timeouts skip the cache so the next request retries instead of being- # pinned to the error for the full 35-minute TTL.+ # pinned to the error for the full TTL.Report.cache(report) if report.error != :timeoutreturn { rows: [], error: report.error.to_s } if report.error.present?diff --git a/spec/jobs/scheduled/warm_dashboard_reports_spec.rb b/spec/jobs/scheduled/warm_dashboard_reports_spec.rbnew file mode 100644index 00000000000..5570bb84a91--- /dev/null+++ b/spec/jobs/scheduled/warm_dashboard_reports_spec.rb@@ -0,0 +1,53 @@+# frozen_string_literal: true++RSpec.describe Jobs::WarmDashboardReports do+ fab!(:admin)++ before do+ Discourse.cache.clear+ SiteSetting.dashboard_improvements = true+ admin.update!(last_seen_at: 1.hour.ago)+ end++ def warmed_signups+ Report.find_cached(+ "signups",+ facets: %i[prev_period],+ start_date: 29.days.ago.to_date.beginning_of_day,+ end_date: Time.zone.now.to_date.end_of_day,+ )+ end++ describe "#execute" do+ it "populates the report cache for the dashboard's default window" do+ described_class.new.execute({})++ expect(warmed_signups).to be_present+ end++ it "warms when only a moderator has been seen recently" do+ admin.update!(last_seen_at: 30.days.ago)+ Fabricate(:moderator, last_seen_at: 1.hour.ago)++ described_class.new.execute({})++ expect(warmed_signups).to be_present+ end++ it "skips warming when the new dashboard is disabled" do+ SiteSetting.dashboard_improvements = false++ described_class.new.execute({})++ expect(warmed_signups).to be_nil+ end++ it "skips warming when no staff member has been seen recently" do+ admin.update!(last_seen_at: 30.days.ago)++ described_class.new.execute({})++ expect(warmed_signups).to be_nil+ end+ end+enddiff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rbindex 9ffd3362999..c2fbad34dc8 100644--- a/spec/models/report_spec.rb+++ b/spec/models/report_spec.rb@@ -1965,11 +1965,11 @@ RSpec.describe Report doReport.cache(exception_report)end- it "caches valid reports for 35 minutes" do+ it "caches valid reports for 60 minutes" doDiscourse.cache.expects(:write)- .with(Report.cache_key(valid_report), valid_report.as_json, expires_in: 35.minutes)+ .with(Report.cache_key(valid_report), valid_report.as_json, expires_in: 60.minutes)Report.cache(valid_report)endenddiff --git a/spec/requests/admin/dashboard_controller_spec.rb b/spec/requests/admin/dashboard_controller_spec.rbindex 813afb817b6..87c221b66b4 100644--- a/spec/requests/admin/dashboard_controller_spec.rb+++ b/spec/requests/admin/dashboard_controller_spec.rb@@ -129,6 +129,12 @@ RSpec.describe Admin::DashboardController doresponse.parsed_body["sections"].index_by { |section| section["id"] }end+ def signups_kpi_value+ highlights =+ response.parsed_body["sections"].find { |section| section["id"] == "highlights" }+ highlights["data"]["kpis"].find { |kpi| kpi["type"] == "new_signups" }["value"]+ end+context "with highlights_data" dolet(:highlights_data) { section_payloads["highlights"]&.dig("data") }@@ -499,6 +505,23 @@ RSpec.describe Admin::DashboardController doendend+ it "serves the default window from the warmed report cache" do+ configure_dashboard_sections(%w[highlights])+ admin.update!(last_seen_at: 1.hour.ago)+ params = { start_date: 29.days.ago.to_date.to_s, end_date: Time.zone.now.to_date.to_s }++ Jobs::WarmDashboardReports.new.execute({})+ Fabricate(:user, created_at: 2.days.ago)++ get "/admin/dashboard.json", params: params+ warmed_signups = signups_kpi_value++ Discourse.cache.clear+ get "/admin/dashboard.json", params: params++ expect(signups_kpi_value).to eq(warmed_signups + 1)+ end+it "denies non-staff users" dosign_in(user)diff --git a/spec/services/admin_dashboard_cache_warmer_spec.rb b/spec/services/admin_dashboard_cache_warmer_spec.rbnew file mode 100644index 00000000000..7154befbe7e--- /dev/null+++ b/spec/services/admin_dashboard_cache_warmer_spec.rb@@ -0,0 +1,52 @@+# frozen_string_literal: true++describe AdminDashboardCacheWarmer do+ before { Discourse.cache.clear }++ def warmed_signups(offset)+ end_date = Time.zone.now.to_date + offset++ Report.find_cached(+ "signups",+ facets: %i[prev_period],+ start_date: (end_date - 29).beginning_of_day,+ end_date: end_date.end_of_day,+ )+ end++ describe ".call" do+ it "warms the default window for the server date and the neighbouring days" do+ described_class.call++ expect(warmed_signups(-1)).to be_present+ expect(warmed_signups(0)).to be_present+ expect(warmed_signups(1)).to be_present+ end++ it "warms the reports backing the highlights and engagement KPIs" do+ described_class.call++ types =+ described_class.report_specs.map do |spec|+ Report.find_cached(spec[:type], spec[:opts].merge(described_class.windows.second))+ end++ expect(types).to all(be_present)+ end+ end++ describe ".report_specs" do+ it "requests the prev_period facet for KPI reports so keys match the dashboard" do+ kpi_specs =+ described_class.report_specs.reject { |spec| spec[:type] == "trust_level_pipeline" }++ expect(kpi_specs.map { |spec| spec[:opts] }.uniq).to eq([{ facets: %i[prev_period] }])+ expect(kpi_specs.map { |spec| spec[:type] }).to include(+ "signups",+ "dau_by_mau",+ "new_contributors",+ "daily_engaged_users",+ )+ end+ end+end