A platform for community discussion. Free, open, simple.
PERF: Reuse cached Data Explorer dashboard reports (#42428)
Krzysztof Kotlarek3a11c435d106
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rbindex d80bb9809b9..80295d2a73b 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rb@@ -3,6 +3,7 @@module DiscourseDataExplorerclass AdminDashboardReportProvider < ::AdminDashboard::Reports::SourceProviderSOURCE_NAME = "data_explorer_query"+ CACHE_MAX_AGE = 1.hourdef self.source_nameSOURCE_NAME@@ -45,7 +46,9 @@ module DiscourseDataExplorerload_queries(identifiers).each_with_object({}) do |query, hash|next if !guardian.user_can_access_query?(query)- result = QueryRunner.run(query, params, current_user: guardian.user)+ result =+ QueryRunner.cached_result(query, params, max_age: CACHE_MAX_AGE) ||+ QueryRunner.run(query, params, current_user: guardian.user)result = result.merge(empty: Array(result[:rows]).empty?) if result.is_a?(Hash)hash[query.id.to_s] = resultenddiff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_cache.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_cache.rbindex 9e734a5bdbc..52dd9530276 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_cache.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_cache.rb@@ -3,7 +3,7 @@module DiscourseDataExplorerclass QueryResultCacheCACHE_TTL = 24.hours.to_i- MAX_CACHE_SIZE = 100.kilobytes+ MAX_CACHE_SIZE = 200.kilobytesMAX_CACHE_ENTRIES = 50def self.cache_key(query_id, params_hash)@@ -14,12 +14,23 @@ module DiscourseDataExplorer"data_explorer:result:#{query_id}:#{digest}"end- def self.read(query_id, params_hash)+ def self.read(query_id, params_hash, max_age: nil)key = cache_key(query_id, params_hash)raw = Discourse.redis.get(key)return nil if raw.nil?- MultiJson.load(raw)++ result = MultiJson.load(raw)+ return nil if max_age && !within_max_age?(result, max_age)++ result+ end++ def self.within_max_age?(result, max_age)+ Time.iso8601(result["cached_at"]) >= max_age.ago+ rescue ArgumentError, TypeError+ falseend+ private_class_method :within_max_age?def self.write(query_id, params_hash, result_json)payload = result_json.merge("cached_at" => Time.now.utc.iso8601)diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rbindex 53e9758ed07..0c1291941e0 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb@@ -23,10 +23,10 @@ module DiscourseDataExplorerresult_jsonend- def self.cached_result(query, raw_params)+ def self.cached_result(query, raw_params, max_age: nil)return nil unless cacheable?(query)params_hash = resolve_params(query, raw_params)- QueryResultCache.read(query.id, params_hash)+ QueryResultCache.read(query.id, params_hash, max_age:)&.deep_symbolize_keysrescue MultiJson::ParseErrornilenddiff --git a/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rb b/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rbindex 6f7afe022a6..dfa4ec2d987 100644--- a/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rb+++ b/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rb@@ -163,6 +163,29 @@ RSpec.describe DiscourseDataExplorer::AdminDashboardReportProvider doexpect(payload[:columns]).to include("value")end+ it "uses cached results that are less than an hour old" do+ DiscourseDataExplorer::QueryRunner.run(visible_query, {}, current_user: admin)++ result = described_class.fetch_many([visible_query.id.to_s], guardian: admin_guardian)+ payload = result[visible_query.id.to_s]++ expect(payload[:cached_at]).to be_present+ expect(payload[:empty]).to eq(false)+ end++ it "runs queries when their cached results are over an hour old" do+ now = Time.now+ freeze_time(now)+ DiscourseDataExplorer::QueryRunner.run(visible_query, {}, current_user: admin)++ freeze_time(now + 1.hour + 1.second)+ result = described_class.fetch_many([visible_query.id.to_s], guardian: admin_guardian)+ payload = result[visible_query.id.to_s]++ expect(payload[:success]).to eq(true)+ expect(payload["cached_at"]).to be_nil+ end+it "skips zero and non-numeric identifiers" doresult = described_class.fetch_many(%w[0 abc], guardian: admin_guardian)expect(result).to be_emptydiff --git a/plugins/discourse-data-explorer/spec/lib/query_result_cache_spec.rb b/plugins/discourse-data-explorer/spec/lib/query_result_cache_spec.rbindex 12421a03476..c793e57bb2d 100644--- a/plugins/discourse-data-explorer/spec/lib/query_result_cache_spec.rb+++ b/plugins/discourse-data-explorer/spec/lib/query_result_cache_spec.rb@@ -36,8 +36,17 @@ describe DiscourseDataExplorer::QueryResultCache doexpect(described_class.read(query.id, params_hash)).to be_nilend+ it "returns nil when the cached result exceeds the maximum age" do+ now = Time.now+ freeze_time(now)+ described_class.write(query.id, params_hash, result_json)++ freeze_time(now + 1.hour + 1.second)+ expect(described_class.read(query.id, params_hash, max_age: 1.hour)).to be_nil+ end+it "skips write when result exceeds max size" do- large_result = result_json.merge("rows" => [["x" * 200_000]])+ large_result = result_json.merge("rows" => [["x" * (described_class::MAX_CACHE_SIZE + 1)]])expect(described_class.write(query.id, params_hash, large_result)).to eq(false)expect(described_class.read(query.id, params_hash)).to be_nildiff --git a/plugins/discourse-data-explorer/spec/lib/query_runner_spec.rb b/plugins/discourse-data-explorer/spec/lib/query_runner_spec.rbindex 60a7c5a7ad0..ae3665cba83 100644--- a/plugins/discourse-data-explorer/spec/lib/query_runner_spec.rb+++ b/plugins/discourse-data-explorer/spec/lib/query_runner_spec.rb@@ -41,7 +41,7 @@ describe DiscourseDataExplorer::QueryRunner docached = described_class.cached_result(query, nil)expect(cached).to be_present- expect(cached["success"]).to eq(true)+ expect(cached[:success]).to eq(true)endit "does not cache when explain is true" do@@ -73,8 +73,8 @@ describe DiscourseDataExplorer::QueryRunner dodescribed_class.run(query, nil, current_user: admin)cached = described_class.cached_result(query, nil)- expect(cached["cached_at"]).to be_present- expect(cached["rows"]).to be_present+ expect(cached[:cached_at]).to be_present+ expect(cached[:rows]).to be_presentendit "returns nil for queries with internal params" dodiff --git a/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb b/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rbindex 0535e935120..3908f8d8572 100644--- a/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb+++ b/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb@@ -594,7 +594,7 @@ describe DiscourseDataExplorer::QueryController docached = DiscourseDataExplorer::QueryRunner.cached_result(query, nil)expect(cached).to be_present- expect(cached["rows"]).to eq([[23]])+ expect(cached[:rows]).to eq([[23]])endit "returns cached results in show response" do