Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Cache data explorer responses by query and param for quick results (#39118)

Natalie Tayb617df4fbd39
diff --git a/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/edit.gjs b/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/edit.gjs
index fb09a4bfaa8..f56fa362afd 100644
--- a/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/edit.gjs
+++ b/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/edit.gjs
@@ -273,6 +273,7 @@ export default class QueriesEdit extends Component {
@showResults={{@controller.showResults}}
@query={{@controller.model}}
@content={{@controller.results}}
+ @cachedAt={{@controller.cachedAt}}
/>
{{/if}}
</div>
diff --git a/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb b/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb
index 1de65640556..59915dbd2d1 100644
--- a/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb
+++ b/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb
@@ -103,7 +103,15 @@ module DiscourseDataExplorer
end
return raise Discourse::NotFound if !guardian.user_can_access_query?(@query) || @query.hidden
- render_serialized @query, QueryDetailsSerializer, root: "query"
+
+ json = serialize_data(@query, QueryDetailsSerializer, root: nil)
+
+ unless params[:export]
+ cached = QueryRunner.cached_result(@query, params[:params])
+ json[:cached_result] = cached if cached
+ end
+
+ render_json_dump(query: json)
end
def groups
@@ -173,6 +181,8 @@ module DiscourseDataExplorer
end
def update
+ sql_changed = @query.sql != params.dig(:query, :sql)
+
ActiveRecord::Base.transaction do
@query.update!(
params.require(:query).permit(:name, :sql, :description).merge(hidden: false),
@@ -183,6 +193,8 @@ module DiscourseDataExplorer
group_ids&.each { |group_id| @query.query_groups.find_or_create_by!(group_id: group_id) }
end
+ QueryRunner.invalidate(@query.id) if sql_changed
+
render_serialized @query, QueryDetailsSerializer, root: "query"
rescue ValidationError => e
render_json_error e.message
@@ -212,75 +224,28 @@ module DiscourseDataExplorer
def run
rate_limit_query_runs!
- check_xhr unless params[:download]
-
query = Query.find(params[:id].to_i)
query.update!(last_run_at: Time.now)
- response.sending_file = true if params[:download]
-
- query_params = {}
- if params[:params]
- query_params =
- params[:params].is_a?(String) ? MultiJson.load(params[:params]) : params[:params]
- end
-
- opts = { current_user: current_user }
- opts[:explain] = true if params[:explain] == "true"
+ explain = params[:explain] == "true"
+ return run_download(query, explain:) if params[:download]
- opts[:limit] = if params[:format] == "csv"
- limit = params.fetch(:limit, QUERY_RESULT_MAX_LIMIT).to_i
- limit = QUERY_RESULT_MAX_LIMIT if limit > QUERY_RESULT_MAX_LIMIT
- limit
- else
+ check_xhr
+ limit =
fetch_limit_from_params(
default: SiteSetting.data_explorer_query_result_limit,
max: QUERY_RESULT_MAX_LIMIT,
)
- end
- result = DataExplorer.run_query(query, query_params, opts)
+ result = QueryRunner.run(query, params[:params], current_user:, explain:, limit:)
if result[:error]
- err = result[:error]
-
- # Pretty printing logic
- err_class = err.class
- err_msg = err.message
- if err.is_a? ActiveRecord::StatementInvalid
- err_class = err.original_exception.class
- err_msg.gsub!("#{err_class}:", "")
- else
- err_msg = "#{err_class}: #{err_msg}"
- end
-
- render json: { success: false, errors: [err_msg] }, status: :unprocessable_entity
+ render json: format_query_error(result[:error]), status: :unprocessable_entity
else
- content_disposition =
- "attachment; filename=#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult"
-
- respond_to do |format|
- format.json do
- response.headers["Content-Disposition"] = "#{content_disposition}.json" if params[
- :download
- ]
-
- render json:
- ResultFormatConverter.convert(
- :json,
- result,
- query_params:,
- download: params[:download],
- explain: params[:explain] == "true",
- )
- end
- format.csv do
- response.headers["Content-Disposition"] = "#{content_disposition}.csv"
-
- render plain: ResultFormatConverter.convert(:csv, result)
- end
- end
+ render json: result
end
+ rescue MultiJson::ParseError
+ render_invalid_json_params
end
private
@@ -301,6 +266,72 @@ module DiscourseDataExplorer
raise e if GlobalSetting.max_data_explorer_api_req_mode.include?("block")
end
+ def run_download(query, explain:)
+ response.sending_file = true
+
+ format = params[:format] == "csv" ? :csv : :json
+ limit =
+ if format == :csv
+ csv_limit = params.fetch(:limit, QUERY_RESULT_MAX_LIMIT).to_i
+ [csv_limit, QUERY_RESULT_MAX_LIMIT].min
+ else
+ fetch_limit_from_params(
+ default: SiteSetting.data_explorer_query_result_limit,
+ max: QUERY_RESULT_MAX_LIMIT,
+ )
+ end
+
+ result =
+ QueryResultDownloader.download(
+ query,
+ params[:params],
+ current_user:,
+ explain:,
+ limit:,
+ format:,
+ )
+
+ if result[:error]
+ return render json: format_query_error(result[:error]), status: :unprocessable_entity
+ end
+
+ content_disposition =
+ "attachment; filename=#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult"
+
+ respond_to do |f|
+ f.json do
+ response.headers["Content-Disposition"] = "#{content_disposition}.json"
+ render json: result[:data]
+ end
+ f.csv do
+ response.headers["Content-Disposition"] = "#{content_disposition}.csv"
+ render plain: result[:data]
+ end
+ end
+ rescue MultiJson::ParseError
+ render_invalid_json_params
+ end
+
+ def format_query_error(err)
+ err_class = err.class
+ err_msg = err.message
+ if err.is_a?(ActiveRecord::StatementInvalid)
+ err_class = err.original_exception.class
+ err_msg.gsub!("#{err_class}:", "")
+ else
+ err_msg = "#{err_class}: #{err_msg}"
+ end
+ { success: false, errors: [err_msg] }
+ end
+
+ def render_invalid_json_params
+ render json: {
+ success: false,
+ errors: ["Invalid JSON in params"],
+ },
+ status: :unprocessable_entity
+ end
+
def set_group
@group = Group.find_by(name: params["group_name"])
end
diff --git a/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-result.gjs b/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-result.gjs
index 3e7893817d3..5e8f9f4aa32 100644
--- a/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-result.gjs
+++ b/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-result.gjs
@@ -4,7 +4,9 @@ import { action } from "@ember/object";
import { schedule } from "@ember/runloop";
import { service } from "@ember/service";
import { capitalize } from "@ember/string";
+import moment from "moment";
import DButton from "discourse/components/d-button";
+import icon from "discourse/helpers/d-icon";
import { ajax } from "discourse/lib/ajax";
import getURL from "discourse/lib/get-url";
import Badge from "discourse/models/badge";
@@ -160,6 +162,15 @@ export default class QueryResult extends Component {
});
}
+ get cachedResultNotice() {
+ if (!this.args.cachedAt) {
+ return null;
+ }
+ return i18n("explorer.cached_result_notice", {
+ relative_time: moment(this.args.cachedAt).fromNow(),
+ });
+ }
+
get parameterAry() {
let arr = [];
for (let key in this.params) {
@@ -370,6 +381,12 @@ export default class QueryResult extends Component {
{{this.resultCount}}
{{this.duration}}
</div>
+ {{#if this.cachedResultNotice}}
+ <div class="cached-result-notice">
+ {{icon "clock-rotate-left"}}
+ {{this.cachedResultNotice}}
+ </div>
+ {{/if}}
<br />
diff --git a/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-results-wrapper.gjs b/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-results-wrapper.gjs
index 2d8ff905804..561d9ed20a4 100644
--- a/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-results-wrapper.gjs
+++ b/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-results-wrapper.gjs
@@ -4,7 +4,11 @@ const QueryResultsWrapper = <template>
{{#if @results}}
<div class="query-results">
{{#if @showResults}}
- <QueryResult @query={{@query}} @content={{@results}} />
+ <QueryResult
+ @query={{@query}}
+ @content={{@results}}
+ @cachedAt={{@cachedAt}}
+ />
{{else}}
{{#each @results.errors as |err|}}
<pre class="query-error"><code>{{~err}}</code></pre>
diff --git a/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/edit.js b/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/edit.js
index 653da3ba232..c57ba0edc87 100644
--- a/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/edit.js
+++ b/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/edit.js
@@ -32,6 +32,7 @@ export default class PluginsExplorerController extends Controller {
@tracked results = this.model.results;
@tracked dirty = false;
@tracked aiGenerating = false;
+ @tracked isCachedResult = false;
queryParams = ["params"];
explain = false;
@@ -52,6 +53,13 @@ export default class PluginsExplorerController extends Controller {
return this.params ? JSON.parse(this.params) : null;
}
+ get cachedAt() {
+ if (this.isCachedResult && this.results?.cached_at) {
+ return this.results.cached_at;
+ }
+ return null;
+ }
+
get editDisabled() {
return this.model.is_default || this.aiGenerating;
}
@@ -342,6 +350,7 @@ export default class PluginsExplorerController extends Controller {
)
.then((result) => {
this.results = result;
+ this.isCachedResult = false;
if (!result.success) {
this.showResults = false;
return;
diff --git a/plugins/discourse-data-explorer/assets/javascripts/discourse/routes/admin-plugins/show/explorer/edit.js b/plugins/discourse-data-explorer/assets/javascripts/discourse/routes/admin-plugins/show/explorer/edit.js
index c83ba419dba..1bfbe327ea1 100644
--- a/plugins/discourse-data-explorer/assets/javascripts/discourse/routes/admin-plugins/show/explorer/edit.js
+++ b/plugins/discourse-data-explorer/assets/javascripts/discourse/routes/admin-plugins/show/explorer/edit.js
@@ -9,7 +9,7 @@ export default class AdminPluginsExplorerQueriesDetails extends DiscourseRoute {
},
};
- model(params) {
+ model(params, transition) {
if (!this.currentUser.admin) {
// display "Only available to admins" message
return { model: null, schema: null, disallow: true, groups: null };
@@ -24,7 +24,16 @@ export default class AdminPluginsExplorerQueriesDetails extends DiscourseRoute {
cache: true,
}
);
- const queryPromise = this.store.find("query", params.query_id);
+
+ const data = {};
+ const urlParams = transition.to.queryParams.params;
+ if (urlParams) {
+ data.params = urlParams;
+ }
+ const queryPromise = ajax(
+ `/admin/plugins/discourse-data-explorer/queries/${params.query_id}`,
+ { data }
+ );
return groupPromise.then((groups) => {
let groupNames = {};
@@ -32,7 +41,8 @@ export default class AdminPluginsExplorerQueriesDetails extends DiscourseRoute {
groupNames[g.id] = g.name;
});
return schemaPromise.then((schema) => {
- return queryPromise.then((model) => {
+ return queryPromise.then((queryResponse) => {
+ const model = this.store.createRecord("query", queryResponse.query);
model.set(
"group_names",
(model.group_ids || []).map((id) => groupNames[id])
@@ -45,13 +55,19 @@ export default class AdminPluginsExplorerQueriesDetails extends DiscourseRoute {
setupController(controller, model, transition) {
controller.teardownAiGeneration();
+
+ const cachedResult = model.model.cached_result;
+ const shouldAutoRun = !!transition.to.queryParams.run;
+ const showCachedResult = !!cachedResult && !shouldAutoRun;
+
controller.setProperties({
...model,
- results: null,
- showResults: false,
+ results: showCachedResult ? cachedResult : null,
+ showResults: showCachedResult,
+ isCachedResult: showCachedResult,
dirty: false,
aiGenerating: false,
- shouldAutoRun: !!transition.to.queryParams.run,
+ shouldAutoRun,
});
controller.setupAiGeneration();
}
diff --git a/plugins/discourse-data-explorer/assets/stylesheets/explorer.scss b/plugins/discourse-data-explorer/assets/stylesheets/explorer.scss
index a866482c38c..a0c8cea0a05 100644
--- a/plugins/discourse-data-explorer/assets/stylesheets/explorer.scss
+++ b/plugins/discourse-data-explorer/assets/stylesheets/explorer.scss
@@ -548,6 +548,13 @@ table.group-reports {
float: right;
}
+.cached-result-notice {
+ color: var(--primary-medium);
+ font-size: var(--font-down-1);
+ clear: both;
+ float: right;
+}
+
.result-explain {
padding-top: 1em;
margin-bottom: 0;
diff --git a/plugins/discourse-data-explorer/config/locales/client.en.yml b/plugins/discourse-data-explorer/config/locales/client.en.yml
index 56d64629b48..0885a2636e8 100644
--- a/plugins/discourse-data-explorer/config/locales/client.en.yml
+++ b/plugins/discourse-data-explorer/config/locales/client.en.yml
@@ -91,6 +91,7 @@ en:
show_graph: "Graph"
others_dirty: "A query has unsaved changes that will be lost if you navigate away."
run_time: "Query completed in %{value} ms."
+ cached_result_notice: "Showing cached results from %{relative_time}."
result_count:
one: "%{count} result."
other: "%{count} results."
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_cache.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_cache.rb
new file mode 100644
index 00000000000..e8cd66d3e65
--- /dev/null
+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_cache.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module DiscourseDataExplorer
+ class QueryResultCache
+ CACHE_TTL = 24.hours.to_i
+ MAX_CACHE_SIZE = 100.kilobytes
+ MAX_CACHE_ENTRIES = 50
+
+ def self.cache_key(query_id, params_hash)
+ h = params_hash || {}
+ h = h.to_unsafe_h if h.respond_to?(:to_unsafe_h)
+ normalized = h.sort.to_h.to_json
+ digest = Digest::SHA256.hexdigest(normalized)
+ "data_explorer:result:#{query_id}:#{digest}"
+ end
+
+ def self.read(query_id, params_hash)
+ key = cache_key(query_id, params_hash)
+ raw = Discourse.redis.get(key)
+ return nil if raw.nil?
+ MultiJson.load(raw)
+ end
+
+ def self.write(query_id, params_hash, result_json)
+ payload = result_json.merge("cached_at" => Time.now.utc.iso8601)
+ serialized = MultiJson.dump(payload)
+ return false if serialized.bytesize > MAX_CACHE_SIZE
+
+ key = cache_key(query_id, params_hash)
+ index_key = cache_index_key(query_id)
+ now = Time.now.to_f
+ return false if limit_reached?(index_key, key)
+
+ Discourse.redis.setex(key, CACHE_TTL, serialized)
+ Discourse.redis.multi do |redis|
+ redis.zadd(index_key, now, key)
+ redis.expire(index_key, CACHE_TTL)
+ end
+ true
+ end
+
+ def self.invalidate(query_id)
+ keys = Discourse.redis.scan_each(match: "data_explorer:result:#{query_id}:*").to_a
+ Discourse.redis.del(*keys) if keys.present?
+ Discourse.redis.del(cache_index_key(query_id))
+ end
+
+ def self.cache_index_key(query_id)
+ "data_explorer:result:#{query_id}:keys"
+ end
+
+ def self.limit_reached?(index_key, key)
+ return false if Discourse.redis.zscore(index_key, key)
+
+ Discourse.redis.zcard(index_key) >= MAX_CACHE_ENTRIES
+ end
+
+ private_class_method :limit_reached?
+ end
+end
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_downloader.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_downloader.rb
new file mode 100644
index 00000000000..a7344b7cadd
--- /dev/null
+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_downloader.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module DiscourseDataExplorer
+ class QueryResultDownloader
+ def self.download(query, raw_params, current_user:, explain: false, limit: nil, format: :json)
+ query_params = QueryRunner.parse_params(raw_params)
+ opts = { current_user: }
+ opts[:explain] = true if explain
+ opts[:limit] = limit if limit
+
+ result = DataExplorer.run_query(query, query_params, opts)
+
+ return { error: result[:error] } if result[:error]
+
+ output =
+ ResultFormatConverter.convert(format, result, query_params:, explain:, download: true)
+
+ { data: output, format: }
+ end
+ end
+end
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb
new file mode 100644
index 00000000000..1bab9b192f5
--- /dev/null
+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module DiscourseDataExplorer
+ class QueryRunner
+ def self.run(query, raw_params, current_user:, explain: false, limit: nil)
+ query_params = parse_params(raw_params)
+ opts = { current_user: }
+ opts[:explain] = true if explain
+ opts[:limit] = limit if limit
+
+ result = DataExplorer.run_query(query, query_params, opts)
+
+ return { error: result[:error], duration_secs: result[:duration_secs] } if result[:error]
+
+ result_json = ResultFormatConverter.convert(:json, result, query_params:, explain:)
+
+ if cacheable?(query, explain:) && default_limit?(limit)
+ cache_key_params = resolve_params(query, raw_params)
+ QueryResultCache.write(query.id, cache_key_params, result_json)
+ end
+
+ result_json
+ end
+
+ def self.cached_result(query, raw_params)
+ return nil unless cacheable?(query)
+ params_hash = resolve_params(query, raw_params)
+ QueryResultCache.read(query.id, params_hash)
+ rescue MultiJson::ParseError
+ nil
+ end
+
+ def self.invalidate(query_id)
+ QueryResultCache.invalidate(query_id)
+ end
+
+ def self.cacheable?(query, explain: false)
+ !explain && !query.params.any?(&:internal?)
+ end
+
+ def self.default_limit?(limit)
+ limit.nil? || limit == SiteSetting.data_explorer_query_result_limit
+ end
+
+ def self.resolve_params(query, raw_params)
+ parsed = parse_params(raw_params)
+ if parsed.present?
+ user_param_ids = query.params.reject(&:internal?).map(&:identifier)
+ parsed.slice(*user_param_ids)
+ else
+ query.params.each_with_object({}) { |p, h| h[p.identifier] = p.default unless p.internal? }
+ end
+ end
+
+ def self.parse_params(raw_params)
+ return {} if raw_params.blank?
+ parsed = raw_params.is_a?(String) ? MultiJson.load(raw_params) : raw_params
+ parsed = parsed.to_unsafe_h if parsed.respond_to?(:to_unsafe_h)
+ parsed.is_a?(Hash) ? parsed : {}
+ end
+
+ private_class_method :resolve_params, :cacheable?, :default_limit?
+ end
+end
diff --git a/plugins/discourse-data-explorer/spec/lib/query_result_cache_spec.rb b/plugins/discourse-data-explorer/spec/lib/query_result_cache_spec.rb
new file mode 100644
index 00000000000..478e663e00a
--- /dev/null
+++ b/plugins/discourse-data-explorer/spec/lib/query_result_cache_spec.rb
@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+describe DiscourseDataExplorer::QueryResultCache do
+ fab!(:query) { Fabricate(:query, sql: "SELECT 1") }
+
+ let(:params_hash) { { "limit" => "10", "name" => "test" } }
+ let(:result_json) do
+ {
+ "success" => true,
+ "columns" => %w[id],
+ "rows" => [[1], [2]],
+ "result_count" => 2,
+ "duration" => 1.5,
+ "params" => params_hash,
+ "relations" => {
+ },
+ "colrender" => {
+ },
+ "default_limit" => 1000,
+ }
+ end
+
+ after { described_class.invalidate(query.id) }
+
+ describe ".write and .read" do
+ it "stores and retrieves results" do
+ described_class.write(query.id, params_hash, result_json)
+ cached = described_class.read(query.id, params_hash)
+
+ expect(cached["success"]).to eq(true)
+ expect(cached["rows"]).to eq([[1], [2]])
+ expect(cached["cached_at"]).to be_present
+ end
+
+ it "returns nil on cache miss" do
+ expect(described_class.read(query.id, params_hash)).to be_nil
+ end
+
+ it "skips write when result exceeds max size" do
+ large_result = result_json.merge("rows" => [["x" * 200_000]])
+
+ expect(described_class.write(query.id, params_hash, large_result)).to eq(false)
+ expect(described_class.read(query.id, params_hash)).to be_nil
+ end
+
+ it "sets a TTL on the cache key" do
+ described_class.write(query.id, params_hash, result_json)
+ key = described_class.cache_key(query.id, params_hash)
+ ttl = Discourse.redis.ttl(key)
+
+ expect(ttl).to be > 0
+ expect(ttl).to be <= described_class::CACHE_TTL
+ end
+
+ it "does not add new cache entries once the per-query limit is reached" do
+ (described_class::MAX_CACHE_ENTRIES + 1).times do |i|
+ described_class.write(query.id, { "value" => i.to_s }, result_json)
+ end
+
+ oldest_key = described_class.cache_key(query.id, { "value" => "0" })
+ overflow_key =
+ described_class.cache_key(query.id, { "value" => described_class::MAX_CACHE_ENTRIES.to_s })
+
+ expect(Discourse.redis.get(oldest_key)).to be_present
+ expect(Discourse.redis.get(overflow_key)).to be_nil
+ expect(Discourse.redis.zcard(described_class.cache_index_key(query.id))).to eq(
+ described_class::MAX_CACHE_ENTRIES,
+ )
+ end
+ end
+
+ describe ".cache_key" do
+ it "produces the same key regardless of param order" do
+ key_a = described_class.cache_key(query.id, { "b" => "2", "a" => "1" })
+ key_b = described_class.cache_key(query.id, { "a" => "1", "b" => "2" })
+
+ expect(key_a).to eq(key_b)
+ end
+
+ it "produces different keys for different params" do
+ key_a = described_class.cache_key(query.id, { "a" => "1" })
+ key_b = described_class.cache_key(query.id, { "a" => "2" })
+
+ expect(key_a).not_to eq(key_b)
+ end
+
+ it "handles nil params" do
+ key = described_class.cache_key(query.id, nil)
+ expect(key).to include("data_explorer:result:#{query.id}:")
+ end
+ end
+
+ describe ".invalidate" do
+ it "removes all cached results for a query" do
+ described_class.write(query.id, { "a" => "1" }, result_json)
+ described_class.write(query.id, { "b" => "2" }, result_json)
+
+ described_class.invalidate(query.id)
+
+ expect(described_class.read(query.id, { "a" => "1" })).to be_nil
+ expect(described_class.read(query.id, { "b" => "2" })).to be_nil
+ end
+
+ it "does not affect other queries" do
+ other_query = Fabricate(:query, sql: "SELECT 2")
+
+ described_class.write(query.id, params_hash, result_json)
+ described_class.write(other_query.id, params_hash, result_json)
+
+ described_class.invalidate(query.id)
+
+ expect(described_class.read(query.id, params_hash)).to be_nil
+ expect(described_class.read(other_query.id, params_hash)).to be_present
+ end
+ end
+end
diff --git a/plugins/discourse-data-explorer/spec/lib/query_result_downloader_spec.rb b/plugins/discourse-data-explorer/spec/lib/query_result_downloader_spec.rb
new file mode 100644
index 00000000000..431b1a19aba
--- /dev/null
+++ b/plugins/discourse-data-explorer/spec/lib/query_result_downloader_spec.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+describe DiscourseDataExplorer::QueryResultDownloader do
+ fab!(:admin)
+ fab!(:query) do
+ Fabricate(
+ :query,
+ sql:
+ "SELECT 1 AS id, 'tomtom' AS name UNION ALL SELECT 2, 'steak' UNION ALL SELECT 3, 'zorro'",
+ user: admin,
+ )
+ end
+
+ describe ".download" do
+ it "returns JSON results" do
+ result = described_class.download(query, nil, current_user: admin, format: :json)
+
+ expect(result[:error]).to be_nil
+ expect(result[:format]).to eq(:json)
+ expect(result[:data][:columns]).to eq(%w[id name])
+ expect(result[:data][:rows]).to eq([[1, "tomtom"], [2, "steak"], [3, "zorro"]])
+ end
+
+ it "returns CSV results" do
+ result = described_class.download(query, nil, current_user: admin, format: :csv)
+
+ expect(result[:error]).to be_nil
+ expect(result[:format]).to eq(:csv)
+
+ rows = result[:data].split("\n")
+ expect(rows[0]).to eq("id,name")
+ expect(rows[1]).to eq("1,tomtom")
+ expect(rows[2]).to eq("2,steak")
+ expect(rows[3]).to eq("3,zorro")
+ end
+
+ it "returns error for invalid queries" do
+ bad_query = Fabricate(:query, sql: "SELECT * FROM nonexistent_table_xyz")
+ result = described_class.download(bad_query, nil, current_user: admin)
+
+ expect(result[:error]).to be_present
+ end
+ end
+end
diff --git a/plugins/discourse-data-explorer/spec/lib/query_runner_spec.rb b/plugins/discourse-data-explorer/spec/lib/query_runner_spec.rb
new file mode 100644
index 00000000000..60a7c5a7ad0
--- /dev/null
+++ b/plugins/discourse-data-explorer/spec/lib/query_runner_spec.rb
@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+describe DiscourseDataExplorer::QueryRunner do
+ fab!(:admin)
+
+ fab!(:query) { Fabricate(:query, sql: "SELECT 1 AS value", user: admin) }
+
+ fab!(:query_with_params) do
+ Fabricate(
+ :query,
+ sql: "-- [params]\n-- int :limit = 10\n\nSELECT 1 AS value LIMIT :limit",
+ user: admin,
+ )
+ end
+
+ fab!(:query_with_internal_params) do
+ Fabricate(
+ :query,
+ sql: "-- [params]\n-- current_user_id :me\n\nSELECT id FROM users WHERE id = :me",
+ user: admin,
+ )
+ end
+
+ after do
+ described_class.invalidate(query.id)
+ described_class.invalidate(query_with_params.id)
+ described_class.invalidate(query_with_internal_params.id)
+ end
+
+ describe ".run" do
+ it "runs the query and returns results" do
+ result = described_class.run(query, nil, current_user: admin)
+
+ expect(result[:success]).to eq(true)
+ expect(result[:columns]).to include("value")
+ expect(result[:rows]).to be_present
+ end
+
+ it "caches the result after execution" do
+ described_class.run(query, nil, current_user: admin)
+
+ cached = described_class.cached_result(query, nil)
+ expect(cached).to be_present
+ expect(cached["success"]).to eq(true)
+ end
+
+ it "does not cache when explain is true" do
+ described_class.run(query, nil, current_user: admin, explain: true)
+
+ expect(described_class.cached_result(query, nil)).to be_nil
+ end
+
+ it "does not cache queries with internal params" do
+ described_class.run(query_with_internal_params, nil, current_user: admin)
+
+ expect(described_class.cached_result(query_with_internal_params, nil)).to be_nil
+ end
+
+ it "returns error info when query fails" do
+ bad_query = Fabricate(:query, sql: "SELECT * FROM nonexistent_table_xyz")
+ result = described_class.run(bad_query, nil, current_user: admin)
+
+ expect(result[:error]).to be_present
+ end
+ end
+
+ describe ".cached_result" do
+ it "returns nil when no cache exists" do
+ expect(described_class.cached_result(query, nil)).to be_nil
+ end
+
+ it "returns cached result after execute" do
+ described_class.run(query, nil, current_user: admin)
+ cached = described_class.cached_result(query, nil)
+
+ expect(cached["cached_at"]).to be_present
+ expect(cached["rows"]).to be_present
+ end
+
+ it "returns nil for queries with internal params" do
+ expect(described_class.cached_result(query_with_internal_params, nil)).to be_nil
+ end
+
+ it "resolves URL params when present" do
+ described_class.run(query_with_params, '{"limit":"5"}', current_user: admin)
+
+ cached = described_class.cached_result(query_with_params, '{"limit":"5"}')
+ expect(cached).to be_present
+
+ cached_default = described_class.cached_result(query_with_params, nil)
+ expect(cached_default).to be_nil
+ end
+
+ it "falls back to default params when no URL params" do
+ described_class.run(query_with_params, '{"limit":"10"}', current_user: admin)
+
+ cached = described_class.cached_result(query_with_params, nil)
+ expect(cached).to be_present
+ end
+
+ it "returns cached result when run was called with null params" do
+ described_class.run(query_with_params, "null", current_user: admin)
+
+ cached = described_class.cached_result(query_with_params, nil)
+ expect(cached).to be_present
+ end
+
+ it "handles malformed JSON params gracefully" do
+ expect(described_class.cached_result(query, "not-json")).to be_nil
+ end
+ end
+
+ describe ".run" do
+ it "does not cache when a non-default limit is used" do
+ described_class.run(query, nil, current_user: admin, limit: 1)
+
+ expect(described_class.cached_result(query, nil)).to be_nil
+ end
+
+ it "caches when using the default limit" do
+ described_class.run(
+ query,
+ nil,
+ current_user: admin,
+ limit: SiteSetting.data_explorer_query_result_limit,
+ )
+
+ expect(described_class.cached_result(query, nil)).to be_present
+ end
+ end
+
+ describe ".invalidate" do
+ it "removes all cached results for a query" do
+ described_class.run(query, nil, current_user: admin)
+ expect(described_class.cached_result(query, nil)).to be_present
+
+ described_class.invalidate(query.id)
+ expect(described_class.cached_result(query, nil)).to be_nil
+ end
+ end
+end
diff --git a/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb b/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb
index f2a7dbef1b5..e048adeb783 100644
--- a/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb
+++ b/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb
@@ -537,6 +537,173 @@ describe DiscourseDataExplorer::QueryController do
end
end
end
+
+ describe "result caching" do
+ def run_query(id, params = {})
+ params = Hash[params.map { |a| [a[0], a[1].to_s] }]
+ post "/admin/plugins/discourse-data-explorer/queries/#{id}/run.json",
+ params: {
+ params: params.to_json,
+ }
+ end
+
+ it "caches results after running a query" do
+ query = make_query("SELECT 23 as my_value")
+
+ run_query query.id
+ expect(response.status).to eq(200)
+
+ cached = DiscourseDataExplorer::QueryRunner.cached_result(query, nil)
+ expect(cached).to be_present
+ expect(cached["rows"]).to eq([[23]])
+ end
+
+ it "returns cached results in show response" do
+ query = make_query("SELECT 23 as my_value")
+ run_query query.id
+
+ get "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json"
+ expect(response.status).to eq(200)
+ expect(response_json["query"]["cached_result"]).to be_present
+ expect(response_json["query"]["cached_result"]["rows"]).to eq([[23]])
+ expect(response_json["query"]["cached_result"]["cached_at"]).to be_present
+ end
+
+ it "returns no cached_result on cache miss" do
+ query = make_query("SELECT 1")
+ get "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json"
+ expect(response.status).to eq(200)
+ expect(response_json["query"]["cached_result"]).to be_nil
+ end
+
+ it "uses URL params for cache lookup" do
+ query = make_query("-- [params]\n-- int :val = 1\n\nSELECT :val as my_value")
+
+ run_query query.id, val: 5
+ run_query query.id, val: 10
+
+ get "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json",
+ params: {
+ params: { val: "5" }.to_json,
+ }
+ expect(response_json["query"]["cached_result"]["rows"]).to eq([[5]])
+
+ get "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json",
+ params: {
+ params: { val: "10" }.to_json,
+ }
+ expect(response_json["query"]["cached_result"]["rows"]).to eq([[10]])
+ end
+
+ it "does not cache results with explain" do
+ query = make_query("SELECT 1")
+ post "/admin/plugins/discourse-data-explorer/queries/#{query.id}/run.json",
+ params: {
+ explain: "true",
+ }
+ expect(response.status).to eq(200)
+
+ cached = DiscourseDataExplorer::QueryRunner.cached_result(query, nil)
+ expect(cached).to be_nil
+ end
+
+ it "does not cache queries with internal params" do
+ query = make_query("-- [params]\n-- current_user_id :me\n\nSELECT :me as user_id")
+
+ run_query query.id
+ expect(response.status).to eq(200)
+
+ cached = DiscourseDataExplorer::QueryRunner.cached_result(query, nil)
+ expect(cached).to be_nil
+ end
+
+ it "invalidates cache when SQL changes" do
+ query = make_query("SELECT 1 as old_value")
+ run_query query.id
+
+ expect(DiscourseDataExplorer::QueryRunner.cached_result(query, nil)).to be_present
+
+ put "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json",
+ params: {
+ query: {
+ name: query.name,
+ sql: "SELECT 2 as new_value",
+ group_ids: [],
+ },
+ }
+ expect(response.status).to eq(200)
+
+ expect(DiscourseDataExplorer::QueryRunner.cached_result(query, nil)).to be_nil
+ end
+
+ it "does not invalidate cache when only name changes" do
+ query = make_query("SELECT 1")
+ run_query query.id
+
+ put "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json",
+ params: {
+ query: {
+ name: "New name",
+ sql: query.sql,
+ group_ids: [],
+ },
+ }
+ expect(response.status).to eq(200)
+
+ expect(DiscourseDataExplorer::QueryRunner.cached_result(query, nil)).to be_present
+ end
+
+ it "returns cached result on reload when run with no explicit params" do
+ query = make_query("-- [params]\n-- int :val = 1\n\nSELECT :val as my_value")
+
+ post "/admin/plugins/discourse-data-explorer/queries/#{query.id}/run.json"
+ expect(response.status).to eq(200)
+
+ get "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json"
+ expect(response_json["query"]["cached_result"]).to be_present
+ end
+
+ it "does not cache results when a non-default limit is used" do
+ query = make_query("SELECT 1")
+ post "/admin/plugins/discourse-data-explorer/queries/#{query.id}/run.json",
+ params: {
+ limit: 1,
+ }
+ expect(response.status).to eq(200)
+
+ cached = DiscourseDataExplorer::QueryRunner.cached_result(query, nil)
+ expect(cached).to be_nil
+ end
+
+ it "handles malformed params in show without error" do
+ query = make_query("SELECT 1")
+ get "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json",
+ params: {
+ params: "not-valid-json",
+ }
+ expect(response.status).to eq(200)
+ expect(response_json["query"]["id"]).to eq(query.id)
+ end
+
+ it "rejects malformed params in run" do
+ query = make_query("SELECT 1")
+ post "/admin/plugins/discourse-data-explorer/queries/#{query.id}/run.json",
+ params: {
+ params: "not-valid-json",
+ }
+ expect(response.status).to eq(422)
+ end
+
+ it "rejects malformed params in download" do
+ query = make_query("SELECT 1")
+ post "/admin/plugins/discourse-data-explorer/queries/#{query.id}/run.json",
+ params: {
+ download: 1,
+ params: "not-valid-json",
+ }
+ expect(response.status).to eq(422)
+ end
+ end
end
describe "Non-Admin" do
diff --git a/plugins/discourse-data-explorer/spec/system/page_objects/pages/data_explorer_query_runner.rb b/plugins/discourse-data-explorer/spec/system/page_objects/pages/data_explorer_query_runner.rb
index 06575fb1b9f..642a3b4e2c1 100644
--- a/plugins/discourse-data-explorer/spec/system/page_objects/pages/data_explorer_query_runner.rb
+++ b/plugins/discourse-data-explorer/spec/system/page_objects/pages/data_explorer_query_runner.rb
@@ -90,6 +90,22 @@ module PageObjects
def has_query_description?(text)
page.has_css?(".query-edit .desc", text: text)
end
+
+ def has_no_result_header?
+ page.has_no_css?(".query-results .result-header")
+ end
+
+ def has_cached_result_notice?
+ page.has_css?(".cached-result-notice")
+ end
+
+ def has_no_cached_result_notice?
+ page.has_no_css?(".cached-result-notice")
+ end
+
+ def has_chart?
+ page.has_css?(".query-results .chart-canvas-container")
+ end
end
end
end
diff --git a/plugins/discourse-data-explorer/spec/system/query_runner_spec.rb b/plugins/discourse-data-explorer/spec/system/query_runner_spec.rb
index 7074dbfea46..a071a9fd396 100644
--- a/plugins/discourse-data-explorer/spec/system/query_runner_spec.rb
+++ b/plugins/discourse-data-explorer/spec/system/query_runner_spec.rb
@@ -88,4 +88,74 @@ RSpec.describe "Data explorer query runner" do
expect(page).to have_current_path("/admin/plugins/discourse-data-explorer/queries")
end
end
+
+ describe "caching results with query params" do
+ fab!(:user_jan) { Fabricate(:user, created_at: Time.parse("2025-01-15")) }
+ fab!(:user_feb) { Fabricate(:user, created_at: Time.parse("2025-02-15")) }
+ fab!(:user_mar) { Fabricate(:user, created_at: Time.parse("2025-03-15")) }
+ fab!(:user_apr) { Fabricate(:user, created_at: Time.parse("2025-04-15")) }
+
+ fab!(:query) { Fabricate(:query, name: "Monthly signups", sql: <<~SQL, user: admin) }
+ -- [params]
+ -- date :start_date = 2025-01-01
+ -- date :end_date = 2025-04-30
+ SELECT
+ date_trunc('month', u.created_at)::date AS month,
+ COUNT(*) AS signups
+ FROM users u
+ WHERE u.created_at >= :start_date::date
+ AND u.created_at < :end_date::date
+ GROUP BY month
+ ORDER BY month
+ SQL
+
+ let(:query_runner) { PageObjects::Pages::DataExplorerQueryRunner.new }
+
+ before do
+ DiscourseDataExplorer::QueryRunner.invalidate(query.id)
+ SiteSetting.data_explorer_enabled = true
+ sign_in admin
+ end
+
+ it "caches results and shows them on reload" do
+ query_runner.visit_admin_query(query.id)
+ expect(query_runner).to have_no_result_header
+
+ query_runner.run_query
+
+ expect(query_runner).to have_result_header
+ expect(query_runner).to have_no_cached_result_notice
+ expect(query_runner).to have_result_row_count(4)
+
+ query_runner.visit_admin_query(
+ query.id,
+ params: {
+ start_date: "2025-01-01",
+ end_date: "2025-03-31",
+ },
+ )
+ expect(query_runner).to have_no_result_header
+
+ query_runner.run_query
+ expect(query_runner).to have_result_header
+ expect(query_runner).to have_result_row_count(3)
+ expect(query_runner).to have_no_cached_result_notice
+
+ query_runner.visit_admin_query(query.id)
+
+ expect(query_runner).to have_result_header
+ expect(query_runner).to have_result_row_count(4)
+ expect(query_runner).to have_cached_result_notice
+ end
+
+ it "forces fresh execution with ?run=1 even when cache exists" do
+ query_runner.visit_admin_query(query.id)
+ query_runner.run_query
+ expect(query_runner).to have_result_header
+
+ query_runner.visit_admin_query(query.id, query_string: "run=true")
+ expect(query_runner).to have_result_header
+ expect(query_runner).to have_no_cached_result_notice
+ end
+ end
end