A platform for community discussion. Free, open, simple.
FEATURE: Cache data explorer responses by query and param for quick results (#39118)
Natalie Tayb617df4fbd39
diff --git a/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/edit.gjs b/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/edit.gjsindex fb09a4bfaa8..f56fa362afd 100644--- a/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/edit.gjs+++ b/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/edit.gjs@@ -273,6 +273,7 @@ export default class QueriesEdit extends Component {@showResults={{@controller.showResults}}@query={{@controller.model}}@content={{@controller.results}}+ @cachedAt={{@controller.cachedAt}}/>{{/if}}</div>diff --git a/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb b/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rbindex 1de65640556..59915dbd2d1 100644--- a/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb+++ b/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb@@ -103,7 +103,15 @@ module DiscourseDataExplorerendreturn raise Discourse::NotFound if !guardian.user_can_access_query?(@query) || @query.hidden- render_serialized @query, QueryDetailsSerializer, root: "query"++ json = serialize_data(@query, QueryDetailsSerializer, root: nil)++ unless params[:export]+ cached = QueryRunner.cached_result(@query, params[:params])+ json[:cached_result] = cached if cached+ end++ render_json_dump(query: json)enddef groups@@ -173,6 +181,8 @@ module DiscourseDataExplorerenddef update+ sql_changed = @query.sql != params.dig(:query, :sql)+ActiveRecord::Base.transaction do@query.update!(params.require(:query).permit(:name, :sql, :description).merge(hidden: false),@@ -183,6 +193,8 @@ module DiscourseDataExplorergroup_ids&.each { |group_id| @query.query_groups.find_or_create_by!(group_id: group_id) }end+ QueryRunner.invalidate(@query.id) if sql_changed+render_serialized @query, QueryDetailsSerializer, root: "query"rescue ValidationError => erender_json_error e.message@@ -212,75 +224,28 @@ module DiscourseDataExplorerdef runrate_limit_query_runs!- check_xhr unless params[:download]-query = Query.find(params[:id].to_i)query.update!(last_run_at: Time.now)- response.sending_file = true if params[:download]-- query_params = {}- if params[:params]- query_params =- params[:params].is_a?(String) ? MultiJson.load(params[:params]) : params[:params]- end-- opts = { current_user: current_user }- opts[:explain] = true if params[:explain] == "true"+ explain = params[:explain] == "true"+ return run_download(query, explain:) if params[:download]- opts[:limit] = if params[:format] == "csv"- limit = params.fetch(:limit, QUERY_RESULT_MAX_LIMIT).to_i- limit = QUERY_RESULT_MAX_LIMIT if limit > QUERY_RESULT_MAX_LIMIT- limit- else+ check_xhr+ limit =fetch_limit_from_params(default: SiteSetting.data_explorer_query_result_limit,max: QUERY_RESULT_MAX_LIMIT,)- end- result = DataExplorer.run_query(query, query_params, opts)+ result = QueryRunner.run(query, params[:params], current_user:, explain:, limit:)if result[:error]- err = result[:error]-- # Pretty printing logic- err_class = err.class- err_msg = err.message- if err.is_a? ActiveRecord::StatementInvalid- err_class = err.original_exception.class- err_msg.gsub!("#{err_class}:", "")- else- err_msg = "#{err_class}: #{err_msg}"- end-- render json: { success: false, errors: [err_msg] }, status: :unprocessable_entity+ render json: format_query_error(result[:error]), status: :unprocessable_entityelse- content_disposition =- "attachment; filename=#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult"-- respond_to do |format|- format.json do- response.headers["Content-Disposition"] = "#{content_disposition}.json" if params[- :download- ]-- render json:- ResultFormatConverter.convert(- :json,- result,- query_params:,- download: params[:download],- explain: params[:explain] == "true",- )- end- format.csv do- response.headers["Content-Disposition"] = "#{content_disposition}.csv"-- render plain: ResultFormatConverter.convert(:csv, result)- end- end+ render json: resultend+ rescue MultiJson::ParseError+ render_invalid_json_paramsendprivate@@ -301,6 +266,72 @@ module DiscourseDataExplorerraise e if GlobalSetting.max_data_explorer_api_req_mode.include?("block")end+ def run_download(query, explain:)+ response.sending_file = true++ format = params[:format] == "csv" ? :csv : :json+ limit =+ if format == :csv+ csv_limit = params.fetch(:limit, QUERY_RESULT_MAX_LIMIT).to_i+ [csv_limit, QUERY_RESULT_MAX_LIMIT].min+ else+ fetch_limit_from_params(+ default: SiteSetting.data_explorer_query_result_limit,+ max: QUERY_RESULT_MAX_LIMIT,+ )+ end++ result =+ QueryResultDownloader.download(+ query,+ params[:params],+ current_user:,+ explain:,+ limit:,+ format:,+ )++ if result[:error]+ return render json: format_query_error(result[:error]), status: :unprocessable_entity+ end++ content_disposition =+ "attachment; filename=#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult"++ respond_to do |f|+ f.json do+ response.headers["Content-Disposition"] = "#{content_disposition}.json"+ render json: result[:data]+ end+ f.csv do+ response.headers["Content-Disposition"] = "#{content_disposition}.csv"+ render plain: result[:data]+ end+ end+ rescue MultiJson::ParseError+ render_invalid_json_params+ end++ def format_query_error(err)+ err_class = err.class+ err_msg = err.message+ if err.is_a?(ActiveRecord::StatementInvalid)+ err_class = err.original_exception.class+ err_msg.gsub!("#{err_class}:", "")+ else+ err_msg = "#{err_class}: #{err_msg}"+ end+ { success: false, errors: [err_msg] }+ end++ def render_invalid_json_params+ render json: {+ success: false,+ errors: ["Invalid JSON in params"],+ },+ status: :unprocessable_entity+ end+def set_group@group = Group.find_by(name: params["group_name"])enddiff --git a/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-result.gjs b/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-result.gjsindex 3e7893817d3..5e8f9f4aa32 100644--- a/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-result.gjs+++ b/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-result.gjs@@ -4,7 +4,9 @@ import { action } from "@ember/object";import { schedule } from "@ember/runloop";import { service } from "@ember/service";import { capitalize } from "@ember/string";+import moment from "moment";import DButton from "discourse/components/d-button";+import icon from "discourse/helpers/d-icon";import { ajax } from "discourse/lib/ajax";import getURL from "discourse/lib/get-url";import Badge from "discourse/models/badge";@@ -160,6 +162,15 @@ export default class QueryResult extends Component {});}+ get cachedResultNotice() {+ if (!this.args.cachedAt) {+ return null;+ }+ return i18n("explorer.cached_result_notice", {+ relative_time: moment(this.args.cachedAt).fromNow(),+ });+ }+get parameterAry() {let arr = [];for (let key in this.params) {@@ -370,6 +381,12 @@ export default class QueryResult extends Component {{{this.resultCount}}{{this.duration}}</div>+ {{#if this.cachedResultNotice}}+ <div class="cached-result-notice">+ {{icon "clock-rotate-left"}}+ {{this.cachedResultNotice}}+ </div>+ {{/if}}<br />diff --git a/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-results-wrapper.gjs b/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-results-wrapper.gjsindex 2d8ff905804..561d9ed20a4 100644--- a/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-results-wrapper.gjs+++ b/plugins/discourse-data-explorer/assets/javascripts/discourse/components/query-results-wrapper.gjs@@ -4,7 +4,11 @@ const QueryResultsWrapper = <template>{{#if @results}}<div class="query-results">{{#if @showResults}}- <QueryResult @query={{@query}} @content={{@results}} />+ <QueryResult+ @query={{@query}}+ @content={{@results}}+ @cachedAt={{@cachedAt}}+ />{{else}}{{#each @results.errors as |err|}}<pre class="query-error"><code>{{~err}}</code></pre>diff --git a/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/edit.js b/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/edit.jsindex 653da3ba232..c57ba0edc87 100644--- a/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/edit.js+++ b/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/edit.js@@ -32,6 +32,7 @@ export default class PluginsExplorerController extends Controller {@tracked results = this.model.results;@tracked dirty = false;@tracked aiGenerating = false;+ @tracked isCachedResult = false;queryParams = ["params"];explain = false;@@ -52,6 +53,13 @@ export default class PluginsExplorerController extends Controller {return this.params ? JSON.parse(this.params) : null;}+ get cachedAt() {+ if (this.isCachedResult && this.results?.cached_at) {+ return this.results.cached_at;+ }+ return null;+ }+get editDisabled() {return this.model.is_default || this.aiGenerating;}@@ -342,6 +350,7 @@ export default class PluginsExplorerController extends Controller {).then((result) => {this.results = result;+ this.isCachedResult = false;if (!result.success) {this.showResults = false;return;diff --git a/plugins/discourse-data-explorer/assets/javascripts/discourse/routes/admin-plugins/show/explorer/edit.js b/plugins/discourse-data-explorer/assets/javascripts/discourse/routes/admin-plugins/show/explorer/edit.jsindex c83ba419dba..1bfbe327ea1 100644--- a/plugins/discourse-data-explorer/assets/javascripts/discourse/routes/admin-plugins/show/explorer/edit.js+++ b/plugins/discourse-data-explorer/assets/javascripts/discourse/routes/admin-plugins/show/explorer/edit.js@@ -9,7 +9,7 @@ export default class AdminPluginsExplorerQueriesDetails extends DiscourseRoute {},};- model(params) {+ model(params, transition) {if (!this.currentUser.admin) {// display "Only available to admins" messagereturn { model: null, schema: null, disallow: true, groups: null };@@ -24,7 +24,16 @@ export default class AdminPluginsExplorerQueriesDetails extends DiscourseRoute {cache: true,});- const queryPromise = this.store.find("query", params.query_id);++ const data = {};+ const urlParams = transition.to.queryParams.params;+ if (urlParams) {+ data.params = urlParams;+ }+ const queryPromise = ajax(+ `/admin/plugins/discourse-data-explorer/queries/${params.query_id}`,+ { data }+ );return groupPromise.then((groups) => {let groupNames = {};@@ -32,7 +41,8 @@ export default class AdminPluginsExplorerQueriesDetails extends DiscourseRoute {groupNames[g.id] = g.name;});return schemaPromise.then((schema) => {- return queryPromise.then((model) => {+ return queryPromise.then((queryResponse) => {+ const model = this.store.createRecord("query", queryResponse.query);model.set("group_names",(model.group_ids || []).map((id) => groupNames[id])@@ -45,13 +55,19 @@ export default class AdminPluginsExplorerQueriesDetails extends DiscourseRoute {setupController(controller, model, transition) {controller.teardownAiGeneration();++ const cachedResult = model.model.cached_result;+ const shouldAutoRun = !!transition.to.queryParams.run;+ const showCachedResult = !!cachedResult && !shouldAutoRun;+controller.setProperties({...model,- results: null,- showResults: false,+ results: showCachedResult ? cachedResult : null,+ showResults: showCachedResult,+ isCachedResult: showCachedResult,dirty: false,aiGenerating: false,- shouldAutoRun: !!transition.to.queryParams.run,+ shouldAutoRun,});controller.setupAiGeneration();}diff --git a/plugins/discourse-data-explorer/assets/stylesheets/explorer.scss b/plugins/discourse-data-explorer/assets/stylesheets/explorer.scssindex a866482c38c..a0c8cea0a05 100644--- a/plugins/discourse-data-explorer/assets/stylesheets/explorer.scss+++ b/plugins/discourse-data-explorer/assets/stylesheets/explorer.scss@@ -548,6 +548,13 @@ table.group-reports {float: right;}+.cached-result-notice {+ color: var(--primary-medium);+ font-size: var(--font-down-1);+ clear: both;+ float: right;+}+.result-explain {padding-top: 1em;margin-bottom: 0;diff --git a/plugins/discourse-data-explorer/config/locales/client.en.yml b/plugins/discourse-data-explorer/config/locales/client.en.ymlindex 56d64629b48..0885a2636e8 100644--- a/plugins/discourse-data-explorer/config/locales/client.en.yml+++ b/plugins/discourse-data-explorer/config/locales/client.en.yml@@ -91,6 +91,7 @@ en:show_graph: "Graph"others_dirty: "A query has unsaved changes that will be lost if you navigate away."run_time: "Query completed in %{value} ms."+ cached_result_notice: "Showing cached results from %{relative_time}."result_count:one: "%{count} result."other: "%{count} results."diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_cache.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_cache.rbnew file mode 100644index 00000000000..e8cd66d3e65--- /dev/null+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_cache.rb@@ -0,0 +1,60 @@+# frozen_string_literal: true++module DiscourseDataExplorer+ class QueryResultCache+ CACHE_TTL = 24.hours.to_i+ MAX_CACHE_SIZE = 100.kilobytes+ MAX_CACHE_ENTRIES = 50++ def self.cache_key(query_id, params_hash)+ h = params_hash || {}+ h = h.to_unsafe_h if h.respond_to?(:to_unsafe_h)+ normalized = h.sort.to_h.to_json+ digest = Digest::SHA256.hexdigest(normalized)+ "data_explorer:result:#{query_id}:#{digest}"+ end++ def self.read(query_id, params_hash)+ key = cache_key(query_id, params_hash)+ raw = Discourse.redis.get(key)+ return nil if raw.nil?+ MultiJson.load(raw)+ end++ def self.write(query_id, params_hash, result_json)+ payload = result_json.merge("cached_at" => Time.now.utc.iso8601)+ serialized = MultiJson.dump(payload)+ return false if serialized.bytesize > MAX_CACHE_SIZE++ key = cache_key(query_id, params_hash)+ index_key = cache_index_key(query_id)+ now = Time.now.to_f+ return false if limit_reached?(index_key, key)++ Discourse.redis.setex(key, CACHE_TTL, serialized)+ Discourse.redis.multi do |redis|+ redis.zadd(index_key, now, key)+ redis.expire(index_key, CACHE_TTL)+ end+ true+ end++ def self.invalidate(query_id)+ keys = Discourse.redis.scan_each(match: "data_explorer:result:#{query_id}:*").to_a+ Discourse.redis.del(*keys) if keys.present?+ Discourse.redis.del(cache_index_key(query_id))+ end++ def self.cache_index_key(query_id)+ "data_explorer:result:#{query_id}:keys"+ end++ def self.limit_reached?(index_key, key)+ return false if Discourse.redis.zscore(index_key, key)++ Discourse.redis.zcard(index_key) >= MAX_CACHE_ENTRIES+ end++ private_class_method :limit_reached?+ end+enddiff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_downloader.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_downloader.rbnew file mode 100644index 00000000000..a7344b7cadd--- /dev/null+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_result_downloader.rb@@ -0,0 +1,21 @@+# frozen_string_literal: true++module DiscourseDataExplorer+ class QueryResultDownloader+ def self.download(query, raw_params, current_user:, explain: false, limit: nil, format: :json)+ query_params = QueryRunner.parse_params(raw_params)+ opts = { current_user: }+ opts[:explain] = true if explain+ opts[:limit] = limit if limit++ result = DataExplorer.run_query(query, query_params, opts)++ return { error: result[:error] } if result[:error]++ output =+ ResultFormatConverter.convert(format, result, query_params:, explain:, download: true)++ { data: output, format: }+ end+ end+enddiff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rbnew file mode 100644index 00000000000..1bab9b192f5--- /dev/null+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/query_runner.rb@@ -0,0 +1,64 @@+# frozen_string_literal: true++module DiscourseDataExplorer+ class QueryRunner+ def self.run(query, raw_params, current_user:, explain: false, limit: nil)+ query_params = parse_params(raw_params)+ opts = { current_user: }+ opts[:explain] = true if explain+ opts[:limit] = limit if limit++ result = DataExplorer.run_query(query, query_params, opts)++ return { error: result[:error], duration_secs: result[:duration_secs] } if result[:error]++ result_json = ResultFormatConverter.convert(:json, result, query_params:, explain:)++ if cacheable?(query, explain:) && default_limit?(limit)+ cache_key_params = resolve_params(query, raw_params)+ QueryResultCache.write(query.id, cache_key_params, result_json)+ end++ result_json+ end++ def self.cached_result(query, raw_params)+ return nil unless cacheable?(query)+ params_hash = resolve_params(query, raw_params)+ QueryResultCache.read(query.id, params_hash)+ rescue MultiJson::ParseError+ nil+ end++ def self.invalidate(query_id)+ QueryResultCache.invalidate(query_id)+ end++ def self.cacheable?(query, explain: false)+ !explain && !query.params.any?(&:internal?)+ end++ def self.default_limit?(limit)+ limit.nil? || limit == SiteSetting.data_explorer_query_result_limit+ end++ def self.resolve_params(query, raw_params)+ parsed = parse_params(raw_params)+ if parsed.present?+ user_param_ids = query.params.reject(&:internal?).map(&:identifier)+ parsed.slice(*user_param_ids)+ else+ query.params.each_with_object({}) { |p, h| h[p.identifier] = p.default unless p.internal? }+ end+ end++ def self.parse_params(raw_params)+ return {} if raw_params.blank?+ parsed = raw_params.is_a?(String) ? MultiJson.load(raw_params) : raw_params+ parsed = parsed.to_unsafe_h if parsed.respond_to?(:to_unsafe_h)+ parsed.is_a?(Hash) ? parsed : {}+ end++ private_class_method :resolve_params, :cacheable?, :default_limit?+ end+enddiff --git a/plugins/discourse-data-explorer/spec/lib/query_result_cache_spec.rb b/plugins/discourse-data-explorer/spec/lib/query_result_cache_spec.rbnew file mode 100644index 00000000000..478e663e00a--- /dev/null+++ b/plugins/discourse-data-explorer/spec/lib/query_result_cache_spec.rb@@ -0,0 +1,116 @@+# frozen_string_literal: true++describe DiscourseDataExplorer::QueryResultCache do+ fab!(:query) { Fabricate(:query, sql: "SELECT 1") }++ let(:params_hash) { { "limit" => "10", "name" => "test" } }+ let(:result_json) do+ {+ "success" => true,+ "columns" => %w[id],+ "rows" => [[1], [2]],+ "result_count" => 2,+ "duration" => 1.5,+ "params" => params_hash,+ "relations" => {+ },+ "colrender" => {+ },+ "default_limit" => 1000,+ }+ end++ after { described_class.invalidate(query.id) }++ describe ".write and .read" do+ it "stores and retrieves results" do+ described_class.write(query.id, params_hash, result_json)+ cached = described_class.read(query.id, params_hash)++ expect(cached["success"]).to eq(true)+ expect(cached["rows"]).to eq([[1], [2]])+ expect(cached["cached_at"]).to be_present+ end++ it "returns nil on cache miss" do+ expect(described_class.read(query.id, params_hash)).to be_nil+ end++ it "skips write when result exceeds max size" do+ large_result = result_json.merge("rows" => [["x" * 200_000]])++ expect(described_class.write(query.id, params_hash, large_result)).to eq(false)+ expect(described_class.read(query.id, params_hash)).to be_nil+ end++ it "sets a TTL on the cache key" do+ described_class.write(query.id, params_hash, result_json)+ key = described_class.cache_key(query.id, params_hash)+ ttl = Discourse.redis.ttl(key)++ expect(ttl).to be > 0+ expect(ttl).to be <= described_class::CACHE_TTL+ end++ it "does not add new cache entries once the per-query limit is reached" do+ (described_class::MAX_CACHE_ENTRIES + 1).times do |i|+ described_class.write(query.id, { "value" => i.to_s }, result_json)+ end++ oldest_key = described_class.cache_key(query.id, { "value" => "0" })+ overflow_key =+ described_class.cache_key(query.id, { "value" => described_class::MAX_CACHE_ENTRIES.to_s })++ expect(Discourse.redis.get(oldest_key)).to be_present+ expect(Discourse.redis.get(overflow_key)).to be_nil+ expect(Discourse.redis.zcard(described_class.cache_index_key(query.id))).to eq(+ described_class::MAX_CACHE_ENTRIES,+ )+ end+ end++ describe ".cache_key" do+ it "produces the same key regardless of param order" do+ key_a = described_class.cache_key(query.id, { "b" => "2", "a" => "1" })+ key_b = described_class.cache_key(query.id, { "a" => "1", "b" => "2" })++ expect(key_a).to eq(key_b)+ end++ it "produces different keys for different params" do+ key_a = described_class.cache_key(query.id, { "a" => "1" })+ key_b = described_class.cache_key(query.id, { "a" => "2" })++ expect(key_a).not_to eq(key_b)+ end++ it "handles nil params" do+ key = described_class.cache_key(query.id, nil)+ expect(key).to include("data_explorer:result:#{query.id}:")+ end+ end++ describe ".invalidate" do+ it "removes all cached results for a query" do+ described_class.write(query.id, { "a" => "1" }, result_json)+ described_class.write(query.id, { "b" => "2" }, result_json)++ described_class.invalidate(query.id)++ expect(described_class.read(query.id, { "a" => "1" })).to be_nil+ expect(described_class.read(query.id, { "b" => "2" })).to be_nil+ end++ it "does not affect other queries" do+ other_query = Fabricate(:query, sql: "SELECT 2")++ described_class.write(query.id, params_hash, result_json)+ described_class.write(other_query.id, params_hash, result_json)++ described_class.invalidate(query.id)++ expect(described_class.read(query.id, params_hash)).to be_nil+ expect(described_class.read(other_query.id, params_hash)).to be_present+ end+ end+enddiff --git a/plugins/discourse-data-explorer/spec/lib/query_result_downloader_spec.rb b/plugins/discourse-data-explorer/spec/lib/query_result_downloader_spec.rbnew file mode 100644index 00000000000..431b1a19aba--- /dev/null+++ b/plugins/discourse-data-explorer/spec/lib/query_result_downloader_spec.rb@@ -0,0 +1,44 @@+# frozen_string_literal: true++describe DiscourseDataExplorer::QueryResultDownloader do+ fab!(:admin)+ fab!(:query) do+ Fabricate(+ :query,+ sql:+ "SELECT 1 AS id, 'tomtom' AS name UNION ALL SELECT 2, 'steak' UNION ALL SELECT 3, 'zorro'",+ user: admin,+ )+ end++ describe ".download" do+ it "returns JSON results" do+ result = described_class.download(query, nil, current_user: admin, format: :json)++ expect(result[:error]).to be_nil+ expect(result[:format]).to eq(:json)+ expect(result[:data][:columns]).to eq(%w[id name])+ expect(result[:data][:rows]).to eq([[1, "tomtom"], [2, "steak"], [3, "zorro"]])+ end++ it "returns CSV results" do+ result = described_class.download(query, nil, current_user: admin, format: :csv)++ expect(result[:error]).to be_nil+ expect(result[:format]).to eq(:csv)++ rows = result[:data].split("\n")+ expect(rows[0]).to eq("id,name")+ expect(rows[1]).to eq("1,tomtom")+ expect(rows[2]).to eq("2,steak")+ expect(rows[3]).to eq("3,zorro")+ end++ it "returns error for invalid queries" do+ bad_query = Fabricate(:query, sql: "SELECT * FROM nonexistent_table_xyz")+ result = described_class.download(bad_query, nil, current_user: admin)++ expect(result[:error]).to be_present+ end+ end+enddiff --git a/plugins/discourse-data-explorer/spec/lib/query_runner_spec.rb b/plugins/discourse-data-explorer/spec/lib/query_runner_spec.rbnew file mode 100644index 00000000000..60a7c5a7ad0--- /dev/null+++ b/plugins/discourse-data-explorer/spec/lib/query_runner_spec.rb@@ -0,0 +1,141 @@+# frozen_string_literal: true++describe DiscourseDataExplorer::QueryRunner do+ fab!(:admin)++ fab!(:query) { Fabricate(:query, sql: "SELECT 1 AS value", user: admin) }++ fab!(:query_with_params) do+ Fabricate(+ :query,+ sql: "-- [params]\n-- int :limit = 10\n\nSELECT 1 AS value LIMIT :limit",+ user: admin,+ )+ end++ fab!(:query_with_internal_params) do+ Fabricate(+ :query,+ sql: "-- [params]\n-- current_user_id :me\n\nSELECT id FROM users WHERE id = :me",+ user: admin,+ )+ end++ after do+ described_class.invalidate(query.id)+ described_class.invalidate(query_with_params.id)+ described_class.invalidate(query_with_internal_params.id)+ end++ describe ".run" do+ it "runs the query and returns results" do+ result = described_class.run(query, nil, current_user: admin)++ expect(result[:success]).to eq(true)+ expect(result[:columns]).to include("value")+ expect(result[:rows]).to be_present+ end++ it "caches the result after execution" do+ described_class.run(query, nil, current_user: admin)++ cached = described_class.cached_result(query, nil)+ expect(cached).to be_present+ expect(cached["success"]).to eq(true)+ end++ it "does not cache when explain is true" do+ described_class.run(query, nil, current_user: admin, explain: true)++ expect(described_class.cached_result(query, nil)).to be_nil+ end++ it "does not cache queries with internal params" do+ described_class.run(query_with_internal_params, nil, current_user: admin)++ expect(described_class.cached_result(query_with_internal_params, nil)).to be_nil+ end++ it "returns error info when query fails" do+ bad_query = Fabricate(:query, sql: "SELECT * FROM nonexistent_table_xyz")+ result = described_class.run(bad_query, nil, current_user: admin)++ expect(result[:error]).to be_present+ end+ end++ describe ".cached_result" do+ it "returns nil when no cache exists" do+ expect(described_class.cached_result(query, nil)).to be_nil+ end++ it "returns cached result after execute" do+ described_class.run(query, nil, current_user: admin)+ cached = described_class.cached_result(query, nil)++ expect(cached["cached_at"]).to be_present+ expect(cached["rows"]).to be_present+ end++ it "returns nil for queries with internal params" do+ expect(described_class.cached_result(query_with_internal_params, nil)).to be_nil+ end++ it "resolves URL params when present" do+ described_class.run(query_with_params, '{"limit":"5"}', current_user: admin)++ cached = described_class.cached_result(query_with_params, '{"limit":"5"}')+ expect(cached).to be_present++ cached_default = described_class.cached_result(query_with_params, nil)+ expect(cached_default).to be_nil+ end++ it "falls back to default params when no URL params" do+ described_class.run(query_with_params, '{"limit":"10"}', current_user: admin)++ cached = described_class.cached_result(query_with_params, nil)+ expect(cached).to be_present+ end++ it "returns cached result when run was called with null params" do+ described_class.run(query_with_params, "null", current_user: admin)++ cached = described_class.cached_result(query_with_params, nil)+ expect(cached).to be_present+ end++ it "handles malformed JSON params gracefully" do+ expect(described_class.cached_result(query, "not-json")).to be_nil+ end+ end++ describe ".run" do+ it "does not cache when a non-default limit is used" do+ described_class.run(query, nil, current_user: admin, limit: 1)++ expect(described_class.cached_result(query, nil)).to be_nil+ end++ it "caches when using the default limit" do+ described_class.run(+ query,+ nil,+ current_user: admin,+ limit: SiteSetting.data_explorer_query_result_limit,+ )++ expect(described_class.cached_result(query, nil)).to be_present+ end+ end++ describe ".invalidate" do+ it "removes all cached results for a query" do+ described_class.run(query, nil, current_user: admin)+ expect(described_class.cached_result(query, nil)).to be_present++ described_class.invalidate(query.id)+ expect(described_class.cached_result(query, nil)).to be_nil+ end+ end+enddiff --git a/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb b/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rbindex f2a7dbef1b5..e048adeb783 100644--- a/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb+++ b/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb@@ -537,6 +537,173 @@ describe DiscourseDataExplorer::QueryController doendendend++ describe "result caching" do+ def run_query(id, params = {})+ params = Hash[params.map { |a| [a[0], a[1].to_s] }]+ post "/admin/plugins/discourse-data-explorer/queries/#{id}/run.json",+ params: {+ params: params.to_json,+ }+ end++ it "caches results after running a query" do+ query = make_query("SELECT 23 as my_value")++ run_query query.id+ expect(response.status).to eq(200)++ cached = DiscourseDataExplorer::QueryRunner.cached_result(query, nil)+ expect(cached).to be_present+ expect(cached["rows"]).to eq([[23]])+ end++ it "returns cached results in show response" do+ query = make_query("SELECT 23 as my_value")+ run_query query.id++ get "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json"+ expect(response.status).to eq(200)+ expect(response_json["query"]["cached_result"]).to be_present+ expect(response_json["query"]["cached_result"]["rows"]).to eq([[23]])+ expect(response_json["query"]["cached_result"]["cached_at"]).to be_present+ end++ it "returns no cached_result on cache miss" do+ query = make_query("SELECT 1")+ get "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json"+ expect(response.status).to eq(200)+ expect(response_json["query"]["cached_result"]).to be_nil+ end++ it "uses URL params for cache lookup" do+ query = make_query("-- [params]\n-- int :val = 1\n\nSELECT :val as my_value")++ run_query query.id, val: 5+ run_query query.id, val: 10++ get "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json",+ params: {+ params: { val: "5" }.to_json,+ }+ expect(response_json["query"]["cached_result"]["rows"]).to eq([[5]])++ get "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json",+ params: {+ params: { val: "10" }.to_json,+ }+ expect(response_json["query"]["cached_result"]["rows"]).to eq([[10]])+ end++ it "does not cache results with explain" do+ query = make_query("SELECT 1")+ post "/admin/plugins/discourse-data-explorer/queries/#{query.id}/run.json",+ params: {+ explain: "true",+ }+ expect(response.status).to eq(200)++ cached = DiscourseDataExplorer::QueryRunner.cached_result(query, nil)+ expect(cached).to be_nil+ end++ it "does not cache queries with internal params" do+ query = make_query("-- [params]\n-- current_user_id :me\n\nSELECT :me as user_id")++ run_query query.id+ expect(response.status).to eq(200)++ cached = DiscourseDataExplorer::QueryRunner.cached_result(query, nil)+ expect(cached).to be_nil+ end++ it "invalidates cache when SQL changes" do+ query = make_query("SELECT 1 as old_value")+ run_query query.id++ expect(DiscourseDataExplorer::QueryRunner.cached_result(query, nil)).to be_present++ put "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json",+ params: {+ query: {+ name: query.name,+ sql: "SELECT 2 as new_value",+ group_ids: [],+ },+ }+ expect(response.status).to eq(200)++ expect(DiscourseDataExplorer::QueryRunner.cached_result(query, nil)).to be_nil+ end++ it "does not invalidate cache when only name changes" do+ query = make_query("SELECT 1")+ run_query query.id++ put "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json",+ params: {+ query: {+ name: "New name",+ sql: query.sql,+ group_ids: [],+ },+ }+ expect(response.status).to eq(200)++ expect(DiscourseDataExplorer::QueryRunner.cached_result(query, nil)).to be_present+ end++ it "returns cached result on reload when run with no explicit params" do+ query = make_query("-- [params]\n-- int :val = 1\n\nSELECT :val as my_value")++ post "/admin/plugins/discourse-data-explorer/queries/#{query.id}/run.json"+ expect(response.status).to eq(200)++ get "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json"+ expect(response_json["query"]["cached_result"]).to be_present+ end++ it "does not cache results when a non-default limit is used" do+ query = make_query("SELECT 1")+ post "/admin/plugins/discourse-data-explorer/queries/#{query.id}/run.json",+ params: {+ limit: 1,+ }+ expect(response.status).to eq(200)++ cached = DiscourseDataExplorer::QueryRunner.cached_result(query, nil)+ expect(cached).to be_nil+ end++ it "handles malformed params in show without error" do+ query = make_query("SELECT 1")+ get "/admin/plugins/discourse-data-explorer/queries/#{query.id}.json",+ params: {+ params: "not-valid-json",+ }+ expect(response.status).to eq(200)+ expect(response_json["query"]["id"]).to eq(query.id)+ end++ it "rejects malformed params in run" do+ query = make_query("SELECT 1")+ post "/admin/plugins/discourse-data-explorer/queries/#{query.id}/run.json",+ params: {+ params: "not-valid-json",+ }+ expect(response.status).to eq(422)+ end++ it "rejects malformed params in download" do+ query = make_query("SELECT 1")+ post "/admin/plugins/discourse-data-explorer/queries/#{query.id}/run.json",+ params: {+ download: 1,+ params: "not-valid-json",+ }+ expect(response.status).to eq(422)+ end+ endenddescribe "Non-Admin" dodiff --git a/plugins/discourse-data-explorer/spec/system/page_objects/pages/data_explorer_query_runner.rb b/plugins/discourse-data-explorer/spec/system/page_objects/pages/data_explorer_query_runner.rbindex 06575fb1b9f..642a3b4e2c1 100644--- a/plugins/discourse-data-explorer/spec/system/page_objects/pages/data_explorer_query_runner.rb+++ b/plugins/discourse-data-explorer/spec/system/page_objects/pages/data_explorer_query_runner.rb@@ -90,6 +90,22 @@ module PageObjectsdef has_query_description?(text)page.has_css?(".query-edit .desc", text: text)end++ def has_no_result_header?+ page.has_no_css?(".query-results .result-header")+ end++ def has_cached_result_notice?+ page.has_css?(".cached-result-notice")+ end++ def has_no_cached_result_notice?+ page.has_no_css?(".cached-result-notice")+ end++ def has_chart?+ page.has_css?(".query-results .chart-canvas-container")+ endendendenddiff --git a/plugins/discourse-data-explorer/spec/system/query_runner_spec.rb b/plugins/discourse-data-explorer/spec/system/query_runner_spec.rbindex 7074dbfea46..a071a9fd396 100644--- a/plugins/discourse-data-explorer/spec/system/query_runner_spec.rb+++ b/plugins/discourse-data-explorer/spec/system/query_runner_spec.rb@@ -88,4 +88,74 @@ RSpec.describe "Data explorer query runner" doexpect(page).to have_current_path("/admin/plugins/discourse-data-explorer/queries")endend++ describe "caching results with query params" do+ fab!(:user_jan) { Fabricate(:user, created_at: Time.parse("2025-01-15")) }+ fab!(:user_feb) { Fabricate(:user, created_at: Time.parse("2025-02-15")) }+ fab!(:user_mar) { Fabricate(:user, created_at: Time.parse("2025-03-15")) }+ fab!(:user_apr) { Fabricate(:user, created_at: Time.parse("2025-04-15")) }++ fab!(:query) { Fabricate(:query, name: "Monthly signups", sql: <<~SQL, user: admin) }+ -- [params]+ -- date :start_date = 2025-01-01+ -- date :end_date = 2025-04-30+ SELECT+ date_trunc('month', u.created_at)::date AS month,+ COUNT(*) AS signups+ FROM users u+ WHERE u.created_at >= :start_date::date+ AND u.created_at < :end_date::date+ GROUP BY month+ ORDER BY month+ SQL++ let(:query_runner) { PageObjects::Pages::DataExplorerQueryRunner.new }++ before do+ DiscourseDataExplorer::QueryRunner.invalidate(query.id)+ SiteSetting.data_explorer_enabled = true+ sign_in admin+ end++ it "caches results and shows them on reload" do+ query_runner.visit_admin_query(query.id)+ expect(query_runner).to have_no_result_header++ query_runner.run_query++ expect(query_runner).to have_result_header+ expect(query_runner).to have_no_cached_result_notice+ expect(query_runner).to have_result_row_count(4)++ query_runner.visit_admin_query(+ query.id,+ params: {+ start_date: "2025-01-01",+ end_date: "2025-03-31",+ },+ )+ expect(query_runner).to have_no_result_header++ query_runner.run_query+ expect(query_runner).to have_result_header+ expect(query_runner).to have_result_row_count(3)+ expect(query_runner).to have_no_cached_result_notice++ query_runner.visit_admin_query(query.id)++ expect(query_runner).to have_result_header+ expect(query_runner).to have_result_row_count(4)+ expect(query_runner).to have_cached_result_notice+ end++ it "forces fresh execution with ?run=1 even when cache exists" do+ query_runner.visit_admin_query(query.id)+ query_runner.run_query+ expect(query_runner).to have_result_header++ query_runner.visit_admin_query(query.id, query_string: "run=true")+ expect(query_runner).to have_result_header+ expect(query_runner).to have_no_cached_result_notice+ end+ endend