Skip to content

A platform for community discussion. Free, open, simple.

PERF: Reuse cached Data Explorer dashboard reports (#42428)

Krzysztof Kotlarek3a11c435d106
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rb
index d80bb9809b9..80295d2a73b 100644
--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rb
+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rb
@@ -3,6 +3,7 @@
module DiscourseDataExplorer
class AdminDashboardReportProvider < ::AdminDashboard::Reports::SourceProvider
SOURCE_NAME = "data_explorer_query"
+ CACHE_MAX_AGE = 1.hour
def self.source_name
SOURCE_NAME
@@ -45,7 +46,9 @@ module DiscourseDataExplorer
load_queries(identifiers).each_with_object({}) do |query, hash|
next if !guardian.user_can_access_query?(query)
- result = QueryRunner.run(query, params, current_user: guardian.user)
+ result =
+ QueryRunner.cached_result(query, params, max_age: CACHE_MAX_AGE) ||
+ QueryRunner.run(query, params, current_user: guardian.user)
result = result.merge(empty: Array(result[:rows]).empty?) if result.is_a?(Hash)
hash[query.id.to_s] = result
end
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_cache.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_cache.rb
index 9e734a5bdbc..52dd9530276 100644
--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_cache.rb
+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_cache.rb
@@ -3,7 +3,7 @@
module DiscourseDataExplorer
class QueryResultCache
CACHE_TTL = 24.hours.to_i
- MAX_CACHE_SIZE = 100.kilobytes
+ MAX_CACHE_SIZE = 200.kilobytes
MAX_CACHE_ENTRIES = 50
def self.cache_key(query_id, params_hash)
@@ -14,12 +14,23 @@ module DiscourseDataExplorer
"data_explorer:result:#{query_id}:#{digest}"
end
- def self.read(query_id, params_hash)
+ def self.read(query_id, params_hash, max_age: nil)
key = cache_key(query_id, params_hash)
raw = Discourse.redis.get(key)
return nil if raw.nil?
- MultiJson.load(raw)
+
+ result = MultiJson.load(raw)
+ return nil if max_age && !within_max_age?(result, max_age)
+
+ result
+ end
+
+ def self.within_max_age?(result, max_age)
+ Time.iso8601(result["cached_at"]) >= max_age.ago
+ rescue ArgumentError, TypeError
+ false
end
+ private_class_method :within_max_age?
def self.write(query_id, params_hash, result_json)
payload = result_json.merge("cached_at" => Time.now.utc.iso8601)
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb
index 53e9758ed07..0c1291941e0 100644
--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb
+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb
@@ -23,10 +23,10 @@ module DiscourseDataExplorer
result_json
end
- def self.cached_result(query, raw_params)
+ def self.cached_result(query, raw_params, max_age: nil)
return nil unless cacheable?(query)
params_hash = resolve_params(query, raw_params)
- QueryResultCache.read(query.id, params_hash)
+ QueryResultCache.read(query.id, params_hash, max_age:)&.deep_symbolize_keys
rescue MultiJson::ParseError
nil
end
diff --git a/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rb b/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rb
index 6f7afe022a6..dfa4ec2d987 100644
--- a/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rb
+++ b/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rb
@@ -163,6 +163,29 @@ RSpec.describe DiscourseDataExplorer::AdminDashboardReportProvider do
expect(payload[:columns]).to include("value")
end
+ it "uses cached results that are less than an hour old" do
+ DiscourseDataExplorer::QueryRunner.run(visible_query, {}, current_user: admin)
+
+ result = described_class.fetch_many([visible_query.id.to_s], guardian: admin_guardian)
+ payload = result[visible_query.id.to_s]
+
+ expect(payload[:cached_at]).to be_present
+ expect(payload[:empty]).to eq(false)
+ end
+
+ it "runs queries when their cached results are over an hour old" do
+ now = Time.now
+ freeze_time(now)
+ DiscourseDataExplorer::QueryRunner.run(visible_query, {}, current_user: admin)
+
+ freeze_time(now + 1.hour + 1.second)
+ result = described_class.fetch_many([visible_query.id.to_s], guardian: admin_guardian)
+ payload = result[visible_query.id.to_s]
+
+ expect(payload[:success]).to eq(true)
+ expect(payload["cached_at"]).to be_nil
+ end
+
it "skips zero and non-numeric identifiers" do
result = described_class.fetch_many(%w[0 abc], guardian: admin_guardian)
expect(result).to be_empty
diff --git a/plugins/discourse-data-explorer/spec/lib/query_result_cache_spec.rb b/plugins/discourse-data-explorer/spec/lib/query_result_cache_spec.rb
index 12421a03476..c793e57bb2d 100644
--- a/plugins/discourse-data-explorer/spec/lib/query_result_cache_spec.rb
+++ b/plugins/discourse-data-explorer/spec/lib/query_result_cache_spec.rb
@@ -36,8 +36,17 @@ describe DiscourseDataExplorer::QueryResultCache do
expect(described_class.read(query.id, params_hash)).to be_nil
end
+ it "returns nil when the cached result exceeds the maximum age" do
+ now = Time.now
+ freeze_time(now)
+ described_class.write(query.id, params_hash, result_json)
+
+ freeze_time(now + 1.hour + 1.second)
+ expect(described_class.read(query.id, params_hash, max_age: 1.hour)).to be_nil
+ end
+
it "skips write when result exceeds max size" do
- large_result = result_json.merge("rows" => [["x" * 200_000]])
+ large_result = result_json.merge("rows" => [["x" * (described_class::MAX_CACHE_SIZE + 1)]])
expect(described_class.write(query.id, params_hash, large_result)).to eq(false)
expect(described_class.read(query.id, params_hash)).to be_nil
diff --git a/plugins/discourse-data-explorer/spec/lib/query_runner_spec.rb b/plugins/discourse-data-explorer/spec/lib/query_runner_spec.rb
index 60a7c5a7ad0..ae3665cba83 100644
--- a/plugins/discourse-data-explorer/spec/lib/query_runner_spec.rb
+++ b/plugins/discourse-data-explorer/spec/lib/query_runner_spec.rb
@@ -41,7 +41,7 @@ describe DiscourseDataExplorer::QueryRunner do
cached = described_class.cached_result(query, nil)
expect(cached).to be_present
- expect(cached["success"]).to eq(true)
+ expect(cached[:success]).to eq(true)
end
it "does not cache when explain is true" do
@@ -73,8 +73,8 @@ describe DiscourseDataExplorer::QueryRunner do
described_class.run(query, nil, current_user: admin)
cached = described_class.cached_result(query, nil)
- expect(cached["cached_at"]).to be_present
- expect(cached["rows"]).to be_present
+ expect(cached[:cached_at]).to be_present
+ expect(cached[:rows]).to be_present
end
it "returns nil for queries with internal params" do
diff --git a/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb b/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb
index 0535e935120..3908f8d8572 100644
--- a/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb
+++ b/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb
@@ -594,7 +594,7 @@ describe DiscourseDataExplorer::QueryController do
cached = DiscourseDataExplorer::QueryRunner.cached_result(query, nil)
expect(cached).to be_present
- expect(cached["rows"]).to eq([[23]])
+ expect(cached[:rows]).to eq([[23]])
end
it "returns cached results in show response" do