A platform for community discussion. Free, open, simple.
PERF: Prewarm pinned Data Explorer dashboard reports (#42429)
Krzysztof Kotlarek92846ee7c95b
diff --git a/app/services/admin_dashboard_cache_warmer.rb b/app/services/admin_dashboard_cache_warmer.rbindex 3ba4ae0feca..e74404d1a34 100644--- a/app/services/admin_dashboard_cache_warmer.rb+++ b/app/services/admin_dashboard_cache_warmer.rb@@ -7,11 +7,12 @@ class AdminDashboardCacheWarmerDATE_OFFSETS = (-1..1)def self.call+ guardian = Guardian.new(Discourse.system_user)+ pinned_reports = AdminDashboard::Reports::Section.build(guardian:)[:items]+windows.each do |window|- report_specs.each do |spec|- report = Report.find(spec[:type], spec[:opts].merge(window))- Report.cache(report) if report && report.error.blank?- end+ warm_core_reports(window)+ warm_pinned_reports(pinned_reports, window, guardian:)endend@@ -36,4 +37,33 @@ class AdminDashboardCacheWarmer{ start_date: (end_date - (WINDOW_DAYS - 1)).beginning_of_day, end_date: end_date.end_of_day }endend++ def self.warm_core_reports(window)+ report_specs.each do |spec|+ report = Report.find(spec[:type], spec[:opts].merge(window))+ Report.cache(report) if report && report.error.blank?+ end+ end+ private_class_method :warm_core_reports++ def self.warm_pinned_reports(reports, window, guardian:)+ return if reports.empty?++ filters = {+ start_date: window[:start_date].to_date.iso8601,+ end_date: window[:end_date].to_date.iso8601,+ }+ AdminDashboard::Reports::Registry.dispatch_per_source(reports) do |provider, group|+ provider.prewarm(group.map { |report| report[:identifier] }, guardian:, filters:)+ rescue StandardError => e+ Discourse.warn_exception(+ e,+ message: "Failed to prewarm dashboard reports",+ env: {+ source: provider.source_name,+ },+ )+ end+ end+ private_class_method :warm_pinned_reportsenddiff --git a/lib/admin_dashboard/reports/source_provider.rb b/lib/admin_dashboard/reports/source_provider.rbindex 4a4526cffce..b1635a0984c 100644--- a/lib/admin_dashboard/reports/source_provider.rb+++ b/lib/admin_dashboard/reports/source_provider.rb@@ -54,6 +54,11 @@ module AdminDashboardraise NotImplementedErrorend+ # Optional cache warm-up hook for reports mounted on the dashboard.+ # Providers that do not cache their payloads can leave this as a no-op.+ def self.prewarm(identifiers, guardian:, filters: {})+ end+# Universe of items of this source. Powers the Manage Reports modal's# available list and search filter. Only invoked in admin-only contexts,# so implementations should not perform per-user access filtering here.diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rbindex 80295d2a73b..f558ca1efcd 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rb@@ -54,6 +54,18 @@ module DiscourseDataExplorerendend+ def self.prewarm(identifiers, guardian:, filters: {})+ return if guardian&.user.nil?++ params = filters.with_indifferent_access+ load_queries(identifiers).each do |query|+ query_params = params.slice(*query.params.map(&:identifier))+ next if !query.groups.empty? || !prewarmable?(query, query_params)++ QueryRunner.run(query, query_params, current_user: guardian.user)+ end+ end+def self.build_resolved(query)::AdminDashboard::Reports::ResolvedReport.new(source: SOURCE_NAME,@@ -92,6 +104,14 @@ module DiscourseDataExplorerendprivate_class_method :load_queries+ def self.prewarmable?(query, params)+ QueryRunner.cacheable?(query) &&+ query.params.all? do |param|+ param.default.present? || param.nullable || params.key?(param.identifier)+ end+ end+ private_class_method :prewarmable?+def self.persisted_after(search:, after:, limit:)scope = Query.where(hidden: false)if search.present?diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rbindex 0c1291941e0..ce8eb4c3035 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb@@ -60,6 +60,6 @@ module DiscourseDataExplorerparsed.is_a?(Hash) ? parsed : {}end- private_class_method :resolve_params, :cacheable?, :default_limit?+ private_class_method :resolve_params, :default_limit?endenddiff --git a/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rb b/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rbindex dfa4ec2d987..47984cd8ec8 100644--- a/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rb+++ b/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rb@@ -31,7 +31,13 @@ RSpec.describe DiscourseDataExplorer::AdminDashboardReportProvider dolet(:user_guardian) { user.guardian }before { SiteSetting.data_explorer_enabled = true }- after { DiscourseDataExplorer::QueryRunner.invalidate(visible_query.id) }+ let(:query_ids_to_invalidate) { [visible_query.id] }++ after do+ query_ids_to_invalidate.each do |query_id|+ DiscourseDataExplorer::QueryRunner.invalidate(query_id)+ end+ enddescribe ".source_name" doit "is 'data_explorer_query'" do@@ -198,6 +204,84 @@ RSpec.describe DiscourseDataExplorer::AdminDashboardReportProvider doendend+ describe ".prewarm" do+ it "warms admin-only queries" do+ described_class.prewarm(+ [visible_query.id.to_s],+ guardian: admin_guardian,+ filters: {+ start_date: "2026-01-01",+ end_date: "2026-01-30",+ },+ )++ cached = DiscourseDataExplorer::QueryRunner.cached_result(visible_query, {})+ expect(cached).to be_present+ end++ it "refreshes cached admin-only queries" do+ now = Time.now+ freeze_time(now)+ DiscourseDataExplorer::QueryRunner.run(visible_query, {}, current_user: admin)++ freeze_time(now + 30.minutes)+ described_class.prewarm([visible_query.id.to_s], guardian: admin_guardian)++ cached = DiscourseDataExplorer::QueryRunner.cached_result(visible_query, {})+ expect(cached[:cached_at]).to eq((now + 30.minutes).utc.iso8601)+ end++ it "uses the cache for queries with default parameters when the dashboard has no filters" do+ query =+ Fabricate(+ :query,+ sql: "-- [params]\n-- int :limit = 10\n\nSELECT :limit AS value",+ user: admin,+ )+ query_ids_to_invalidate << query.id++ described_class.prewarm(+ [query.id.to_s],+ guardian: admin_guardian,+ filters: {+ start_date: "2026-01-01",+ end_date: "2026-01-30",+ },+ )++ result = described_class.fetch_many([query.id.to_s], guardian: admin_guardian, filters: {})+ expect(result[query.id.to_s][:cached_at]).to be_present+ end++ it "skips queries shared with a group" do+ DiscourseDataExplorer::QueryGroup.create!(query: visible_query, group: group)++ described_class.prewarm([visible_query.id.to_s], guardian: admin_guardian)++ expect(DiscourseDataExplorer::QueryRunner.cached_result(visible_query, {})).to be_nil+ end++ it "skips queries with required parameters outside the dashboard filters" do+ query =+ Fabricate(+ :query,+ sql: "-- [params]\n-- int :topic_id\n\nSELECT :topic_id AS value",+ user: admin,+ )++ described_class.prewarm(+ [query.id.to_s],+ guardian: admin_guardian,+ filters: {+ start_date: "2026-01-01",+ end_date: "2026-01-30",+ },+ )++ expect(DiscourseDataExplorer::QueryRunner.cached_result(query, {})).to be_nil+ end+ end+describe "registration" doit "is registered as a provider on boot" doexpect(AdminDashboard::Reports::Registry.provider_for("data_explorer_query")).to eq(diff --git a/spec/services/admin_dashboard_cache_warmer_spec.rb b/spec/services/admin_dashboard_cache_warmer_spec.rbindex 7154befbe7e..4920a2b6ae1 100644--- a/spec/services/admin_dashboard_cache_warmer_spec.rb+++ b/spec/services/admin_dashboard_cache_warmer_spec.rb@@ -3,6 +3,54 @@describe AdminDashboardCacheWarmer dobefore { Discourse.cache.clear }+ let(:prewarm_calls) { [] }+ let(:prewarming_provider) do+ calls = prewarm_calls+ Class.new(AdminDashboard::Reports::SourceProvider) do+ def self.source_name = "prewarming_source"++ define_singleton_method(:resolve_many) do |identifiers, guardian:|+ next {} if guardian.nil?++ identifiers.each_with_object({}) do |identifier, reports|+ reports[identifier] = AdminDashboard::Reports::ResolvedReport.new(+ source: source_name,+ identifier:,+ title: identifier,+ description: nil,+ label: nil,+ url: "",+ )+ end+ end++ define_singleton_method(:prewarm) do |identifiers, guardian:, filters:|+ calls << { identifiers:, guardian:, filters: }+ end+ end+ end+ let(:failing_provider) do+ Class.new(prewarming_provider) do+ def self.source_name = "failing_source"++ def self.prewarm(...)+ raise StandardError, "prewarm failed"+ end+ end+ end+ let(:plugin) { Plugin::Instance.new }++ before do+ DiscoursePluginRegistry.register_admin_dashboard_report_source(prewarming_provider, plugin)+ DiscoursePluginRegistry.register_admin_dashboard_report_source(failing_provider, plugin)+ end++ after do+ DiscoursePluginRegistry._raw_admin_dashboard_report_sources.reject! do |entry|+ [prewarming_provider, failing_provider].include?(entry[:value])+ end+ end+def warmed_signups(offset)end_date = Time.zone.now.to_date + offset@@ -33,6 +81,41 @@ describe AdminDashboardCacheWarmer doexpect(types).to all(be_present)end++ it "asks report providers to warm their pinned reports for each default window" do+ AdminDashboardReport.create!(+ source: prewarming_provider.source_name,+ identifier: "report-id",+ position: 0,+ )++ described_class.call++ expect(prewarm_calls.map { |call| call[:identifiers] }).to eq(+ Array.new(described_class.windows.size, ["report-id"]),+ )+ expect(prewarm_calls.map { |call| call[:guardian].user }).to all(eq(Discourse.system_user))+ expect(prewarm_calls.map { |call| call[:filters] }).to eq(+ described_class.windows.map do |window|+ {+ start_date: window[:start_date].to_date.iso8601,+ end_date: window[:end_date].to_date.iso8601,+ }+ end,+ )+ end++ it "continues warming core reports when a pinned provider raises" do+ AdminDashboardReport.create!(+ source: failing_provider.source_name,+ identifier: "report-id",+ position: 0,+ )++ described_class.call++ expect([warmed_signups(-1), warmed_signups(0), warmed_signups(1)]).to all(be_present)+ endenddescribe ".report_specs" do