A platform for community discussion. Free, open, simple.
DEV: Include default Data Explorer queries in JSON:API reads
Loïc Guitaut141c29600f47
diff --git a/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb b/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rbindex 58e98b6184e..de779e11341 100644--- a/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb+++ b/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb@@ -35,6 +35,7 @@ module DiscourseDataExplorerelsebase_scope = base_scope.order(order_column => order_direction)end+ base_scope = base_scope.order(:id)if filter.present?sanitized_filter = "%#{Query.sanitize_sql_like(filter)}%"diff --git a/plugins/discourse-data-explorer/app/models/discourse_data_explorer/query.rb b/plugins/discourse-data-explorer/app/models/discourse_data_explorer/query.rbindex 48adc7432ae..fb41ef00ea1 100644--- a/plugins/discourse-data-explorer/app/models/discourse_data_explorer/query.rb+++ b/plugins/discourse-data-explorer/app/models/discourse_data_explorer/query.rb@@ -130,7 +130,7 @@ module DiscourseDataExplorerdef self.unpersisted_defaults(search: nil, tags: nil, tag: nil)selected_tags = QueryTag.normalize_all(tags || tag)- persisted_ids = where(hidden: false).where("id < 0").pluck(:id).to_set+ persisted_ids = where("id < 0").pluck(:id).to_setquery_text = search&.downcaseQueries.default.filter_map do |_, attributes|diff --git a/plugins/discourse-data-explorer/db/fixtures/001_default_queries.rb b/plugins/discourse-data-explorer/db/fixtures/001_default_queries.rbnew file mode 100644index 00000000000..a56be53ec9f--- /dev/null+++ b/plugins/discourse-data-explorer/db/fixtures/001_default_queries.rb@@ -0,0 +1,3 @@+# frozen_string_literal: true++DiscourseDataExplorer::DefaultQuerySync.execute unless Rails.env.test?diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/default_query_sync.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/default_query_sync.rbnew file mode 100644index 00000000000..7780b8c4c81--- /dev/null+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/default_query_sync.rb@@ -0,0 +1,33 @@+# frozen_string_literal: true++module DiscourseDataExplorer+ class DefaultQuerySync+ def self.execute+ Queries.default.each_value { new(it).execute }+ end++ def initialize(definition)+ @attributes = definition.merge("user_id" => Discourse::SYSTEM_USER_ID)+ end++ def execute+ query.with_lock do+ query.update!(attributes)+ invalidate_cached_results if query.saved_change_to_sql?+ end+ end++ private++ attr_reader :attributes++ def query+ @query ||=+ Query.find_or_create_by!(id: attributes.fetch("id")) { it.assign_attributes(attributes) }+ end++ def invalidate_cached_results+ ActiveRecord.after_all_transactions_commit { QueryRunner.invalidate(query.id) }+ end+ end+enddiff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rbindex da1f3ae0e18..6fcfa525223 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb@@ -15,10 +15,7 @@ module DiscourseDataExplorer# WARNING: Edit the query hash carefully# For each query, add id, name, description and optional tags here and add sql below# Feel free to add new queries at the bottom of the hash in numerical order- # If any query has been run on an instance, it is then saved in the local db- # Locally stored queries are updated from the below data only when they are run again- # eg. If you update a query with id=-1 in this file and the query has been run on a site,- # you must run the query with id=-1 on the site again to update these changes in the site db+ # The seed task synchronizes these definitions without changing local query state.queries = {"most-common-likers": {diff --git a/plugins/discourse-data-explorer/plugin.rb b/plugins/discourse-data-explorer/plugin.rbindex 4c1513a8db2..96aeab2e7d3 100644--- a/plugins/discourse-data-explorer/plugin.rb+++ b/plugins/discourse-data-explorer/plugin.rb@@ -34,6 +34,8 @@ endrequire_relative "lib/discourse_data_explorer/engine"after_initialize do+ register_seedfu_fixtures(Rails.root.join("plugins/discourse-data-explorer/db/fixtures"))+require_relative "lib/discourse_data_explorer/mcp_tools"register_mcp_tool(diff --git a/plugins/discourse-data-explorer/spec/lib/default_query_sync_spec.rb b/plugins/discourse-data-explorer/spec/lib/default_query_sync_spec.rbnew file mode 100644index 00000000000..730e5f66f92--- /dev/null+++ b/plugins/discourse-data-explorer/spec/lib/default_query_sync_spec.rb@@ -0,0 +1,166 @@+# frozen_string_literal: true++RSpec.describe DiscourseDataExplorer::DefaultQuerySync do+ fab!(:admin)+ fab!(:group)+ fab!(:custom_query, :query)++ let(:definition) do+ { "id" => -1, "name" => "Default query", "description" => "A report", "sql" => "SELECT 1" }+ end+ let(:query) { DiscourseDataExplorer::Query.find_by(id: -1) }+ let(:cache) { DiscourseDataExplorer::QueryResultCache }++ describe ".execute" do+ subject(:sync) { described_class.execute }++ let(:other_definition) { definition.merge("id" => -2, "name" => "Other default") }+ let(:definitions) { { "-1" => definition, "-2" => other_definition } }++ before { allow(DiscourseDataExplorer::Queries).to receive(:default).and_return(definitions) }++ it "synchronizes each default query" do+ sync++ expect(+ DiscourseDataExplorer::Query.where("id < 0").map { it.attributes.slice(*definition.keys) },+ ).to contain_exactly(definition, other_definition)+ end++ context "when database seeds run" do+ subject(:sync) do+ silence_stdout do+ SeedFu.seed(DiscoursePluginRegistry.seed_paths, %r{discourse-data-explorer/db/fixtures/})+ end+ end++ before { allow(Rails.env).to receive(:test?).and_return(false) }++ it "synchronizes default queries" do+ expect { sync }.to change { DiscourseDataExplorer::Query.exists?(id: -1) }.to(true)+ end+ end++ context "when defaults are already synchronized" do+ before do+ described_class.execute+ freeze_time 1.day.from_now+ end++ it "leaves the records unchanged" do+ expect { sync }.not_to change { DiscourseDataExplorer::Query.order(:id).map(&:attributes) }+ end+ end++ context "when a new default query is added" do+ let(:new_definition) { definition.merge("id" => -3, "name" => "New default") }++ before do+ described_class.execute+ allow(DiscourseDataExplorer::Queries).to receive(:default).and_return(+ definitions.merge("-3" => new_definition),+ )+ end++ it "persists the new definition" do+ expect { sync }.to change { DiscourseDataExplorer::Query.find_by(id: -3)&.attributes }.to(+ a_hash_including(new_definition),+ )+ end+ end+ end++ describe "#execute" do+ subject(:sync) { described_class.new(definition).execute }++ it "persists the default definition" do+ sync++ expect(query).to have_attributes(+ definition.merge("user_id" => Discourse::SYSTEM_USER_ID, "last_run_at" => nil),+ )+ end++ it "preserves custom queries" do+ expect { sync }.not_to change { custom_query.reload.attributes }+ end++ context "when a default query already exists" do+ let!(:existing_query) do+ Fabricate(+ :query,+ id: -1,+ sql: "SELECT 0",+ hidden: true,+ last_run_at: 2.days.ago,+ created_at: 1.year.ago,+ )+ end+ let(:local_state) { %w[hidden last_run_at created_at] }++ before do+ existing_query.groups = [group]+ DiscourseDataExplorer::QueryTag.sync!(query: existing_query, names: ["staff"])+ cache.write(existing_query.id, admin, {}, "rows" => [[0]])+ end++ it "refreshes the shipped definition" do+ expect { sync }.to change { existing_query.reload.attributes.slice(*definition.keys) }.to(+ definition,+ )+ end++ it "preserves local query state" do+ expect { sync }.not_to change { existing_query.reload.attributes.slice(*local_state) }+ end++ it "preserves group access" do+ expect { sync }.not_to change { existing_query.reload.group_ids }+ end++ it "preserves local tags" do+ expect { sync }.not_to change { existing_query.reload.tags.pluck(:name) }+ end++ it "invalidates cached results from the previous SQL" do+ expect { sync }.to change { cache.read(existing_query.id, admin, {}) }.to(nil)+ end++ context "when the SQL is unchanged" do+ let(:definition) { super().merge("sql" => "SELECT 0") }++ it "preserves cached results" do+ expect { sync }.not_to change { cache.read(existing_query.id, admin, {}) }+ end+ end++ context "when the definition is invalid" do+ let(:definition) { super().merge("name" => nil) }++ it "preserves the stored definition", :aggregate_failures do+ expect { sync }.to raise_error(ActiveRecord::RecordInvalid)+ expect(existing_query.reload.sql).to eq("SELECT 0")+ end++ it "preserves cached results", :aggregate_failures do+ expect { sync }.to raise_error(ActiveRecord::RecordInvalid)+ expect(cache.read(existing_query.id, admin, {})).to include("rows" => [[0]])+ end+ end++ context "when synchronization rolls back" do+ subject(:sync) do+ DiscourseDataExplorer::Query.transaction do+ described_class.new(definition).execute+ raise ActiveRecord::Rollback+ end+ end++ it "preserves the definition and cached results", :aggregate_failures do+ expect { sync }.not_to change { cache.read(existing_query.id, admin, {}) }+ expect(existing_query.reload.sql).to eq("SELECT 0")+ end+ end+ end+ end+enddiff --git a/plugins/discourse-data-explorer/spec/requests/api/queries_spec.rb b/plugins/discourse-data-explorer/spec/requests/api/queries_spec.rbindex 57d37d160a9..5e81bbaa12f 100644--- a/plugins/discourse-data-explorer/spec/requests/api/queries_spec.rb+++ b/plugins/discourse-data-explorer/spec/requests/api/queries_spec.rb@@ -36,8 +36,6 @@ RSpec.describe "JSON:API queries", type: :request dofab!(:analysts) { Fabricate(:group, name: "analysts") }- before { [oldest, middle, newest].each { Fabricate(:query_group, query: it, group: analysts) } }-let(:base) { "http://test.localhost/api/data-explorer" }let(:root) { "http://test.localhost/api" }let(:query_parameters) { {} }@@ -47,8 +45,23 @@ RSpec.describe "JSON:API queries", type: :request dobefore doSiteSetting.data_explorer_enabled = true+ host! Discourse.current_hostnamesign_in(user)- get path, headers:, params: query_parameters+ [oldest, middle, newest].each { Fabricate(:query_group, query: it, group: analysts) }+ end++ shared_context "with synchronized default queries" do+ let!(:existing_query) do+ Fabricate(:query, id: -1, name: "Old definition", hidden:, last_run_at: 2.days.ago)+ end+ let(:hidden) { false }+ let(:definition) { DiscourseDataExplorer::Queries.default.fetch("-1") }++ before do+ existing_query.groups = [analysts]+ DiscourseDataExplorer::QueryTag.sync!(query: existing_query, names: ["staff"])+ DiscourseDataExplorer::DefaultQuerySync.execute+ endendshared_examples "a request only for admins" do@@ -103,108 +116,175 @@ RSpec.describe "JSON:API queries", type: :request do}end- it_behaves_like "a request only for admins"+ context "when queries are custom" do+ before { get(path, headers:, params: query_parameters) }- it "sends the first page of queries" do- expect(parsed_body).to eq("data" => data, "included" => [], "links" => links)- end+ it_behaves_like "a request only for admins"- context "when the request filters by a word in the name" do- let(:query_parameters) { { filter: { search: "NEVER POSTED" } } }+ it "sends the first page of queries" do+ expect(parsed_body).to eq("data" => data, "included" => [], "links" => links)+ end- it "sends the queries that match, whatever the case" do- expect(parsed_body["data"].map { it["attributes"]["name"] }).to eq(- ["Users who never posted"],- )+ context "when the request filters by a word in the name" do+ let(:query_parameters) { { filter: { search: "NEVER POSTED" } } }++ it "sends the queries that match, whatever the case" do+ expect(parsed_body["data"].map { it["attributes"]["name"] }).to eq(+ ["Users who never posted"],+ )+ endend- end- context "when the request filters by a word in the description" do- let(:query_parameters) { { filter: { search: "seven days" } } }+ context "when the request filters by a word in the description" do+ let(:query_parameters) { { filter: { search: "seven days" } } }- it "sends the queries that match" do- expect(parsed_body["data"].map { it["attributes"]["name"] }).to eq(["Posts of the week"])+ it "sends the queries that match" do+ expect(parsed_body["data"].map { it["attributes"]["name"] }).to eq(["Posts of the week"])+ end+ end++ context "when the filter value holds a wildcard character" do+ let(:query_parameters) { { filter: { search: "%" } } }++ it "reads it as a plain character" do+ expect(parsed_body["data"]).to be_empty+ end+ end++ context "when the request includes the author and the groups" do+ let(:query_parameters) { { include: "user,groups" } }++ it "sends every related record under its own namespace" do+ expect(parsed_body["included"]).to contain_exactly(+ {+ "type" => "users",+ "id" => author.id.to_s,+ "attributes" => {+ "username" => "query_author",+ },+ "links" => {+ "self" => "#{root}/users/#{author.id}",+ },+ },+ {+ "type" => "groups",+ "id" => analysts.id.to_s,+ "attributes" => {+ "name" => "analysts",+ },+ "links" => {+ "self" => "#{root}/groups/#{analysts.id}",+ },+ },+ )+ endendend- context "when the filter value holds a wildcard character" do- let(:query_parameters) { { filter: { search: "%" } } }+ context "when default queries are synchronized" do+ include_context "with synchronized default queries"++ let(:query_parameters) { super().merge(sort: "name") }+ let(:query_ids) { DiscourseDataExplorer::Query.where(hidden: false).order(:name).pluck(:id) }+ let(:first_page) { JSON.parse(response.body) }+ let(:second_page) { JSON.parse(response.body) }+ let(:first_page_ids) { first_page.fetch("data").map { it["id"].to_i } }+ let(:second_page_ids) { second_page.fetch("data").map { it["id"].to_i } }++ before { get(path, headers:, params: query_parameters) }++ it "paginates default and custom queries together", :aggregate_failures do+ expect(first_page_ids.size).to eq(50)- it "reads it as a plain character" do- expect(parsed_body["data"]).to be_empty+ get(first_page.fetch("links").fetch("next"), headers:)++ expect(first_page_ids + second_page_ids).to eq(query_ids)+ expect(second_page.dig("links", "next")).to be_nil+ end++ context "when a default query is hidden" do+ let(:hidden) { true }++ it "excludes the query from the list" do+ expect(first_page_ids).not_to include(existing_query.id)+ endendend+ end- context "when the request includes the author and the groups" do- let(:query_parameters) { { include: "user,groups" } }+ describe "GET /show" do+ let(:path) { "/api/data-explorer/queries/#{middle.id}" }- it "sends every related record under its own namespace" do- expect(parsed_body["included"]).to contain_exactly(- {- "type" => "users",- "id" => author.id.to_s,+ context "when queries are custom" do+ before { get(path, headers:, params: query_parameters) }++ it_behaves_like "a request only for admins"++ it "sends one query" do+ expect(parsed_body).to eq(+ "data" => {+ "type" => "queries",+ "id" => middle.id.to_s,"attributes" => {- "username" => "query_author",+ "name" => middle.name,+ "description" => middle.description,+ "sql" => middle.sql,+ "paramInfo" => [],+ "isDefault" => false,+ "createdAt" => middle.created_at.as_json,+ "lastRunAt" => middle.last_run_at.as_json,+ "tags" => ["custom"],},"links" => {- "self" => "#{root}/users/#{author.id}",+ "self" => "#{base}/queries/#{middle.id}",},},- {- "type" => "groups",- "id" => analysts.id.to_s,- "attributes" => {- "name" => "analysts",- },- "links" => {- "self" => "#{root}/groups/#{analysts.id}",+ "included" => [],+ "links" => {+ "self" => {+ "href" => "#{base}/queries/#{middle.id}",+ "type" => JsonApiKit::Pagination::Profile::MEDIA_TYPE,},},)end- end- end- describe "GET /show" do- let(:path) { "/api/data-explorer/queries/#{middle.id}" }+ context "when the query is hidden" do+ fab!(:secret) { Fabricate(:query, name: "Hidden from the listing", hidden: true) }- it_behaves_like "a request only for admins"+ let(:path) { "/api/data-explorer/queries/#{secret.id}" }- it "sends one query" do- expect(parsed_body).to eq(- "data" => {- "type" => "queries",- "id" => middle.id.to_s,- "attributes" => {- "name" => middle.name,- "description" => middle.description,- "sql" => middle.sql,- "paramInfo" => [],- "isDefault" => false,- "createdAt" => middle.created_at.as_json,- "lastRunAt" => middle.last_run_at.as_json,- "tags" => ["custom"],- },- "links" => {- "self" => "#{base}/queries/#{middle.id}",- },- },- "included" => [],- "links" => {- "self" => {- "href" => "#{base}/queries/#{middle.id}",- "type" => JsonApiKit::Pagination::Profile::MEDIA_TYPE,- },- },- )+ it { expect(response).to have_http_status(:not_found) }+ endend- context "when the query is hidden" do- fab!(:secret) { Fabricate(:query, name: "Hidden from the listing", hidden: true) }+ context "when default queries are synchronized" do+ include_context "with synchronized default queries"++ let(:path) { "/api/data-explorer/queries/#{existing_query.id}" }- let(:path) { "/api/data-explorer/queries/#{secret.id}" }+ before { get(path, headers:, params: query_parameters) }- it { expect(response).to have_http_status(:not_found) }+ it "serves the current default definition" do+ expect(parsed_body.fetch("data")).to include(+ "id" => existing_query.id.to_s,+ "attributes" =>+ a_hash_including(+ "name" => definition.fetch("name"),+ "sql" => definition.fetch("sql"),+ "isDefault" => true,+ "tags" => %w[default users staff],+ ),+ )+ end++ context "when a default query is hidden" do+ let(:hidden) { true }++ it "returns not found for the query" do+ expect(response).to have_http_status(:not_found)+ end+ endendendenddiff --git a/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb b/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rbindex cbf4fbc0dfd..2af8c90f142 100644--- a/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb+++ b/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-describe DiscourseDataExplorer::QueryController do+RSpec.describe DiscourseDataExplorer::QueryController dodef response_jsonresponse.parsed_bodyend@@ -34,6 +34,22 @@ describe DiscourseDataExplorer::QueryController do}end+ shared_context "with synchronized default queries" do+ fab!(:group)++ let!(:existing_query) do+ Fabricate(:query, id: -1, name: "Old definition", hidden:, last_run_at: 2.days.ago)+ end+ let(:hidden) { false }+ let(:definition) { DiscourseDataExplorer::Queries.default.fetch("-1") }++ before do+ existing_query.groups = [group]+ DiscourseDataExplorer::QueryTag.sync!(query: existing_query, names: ["staff"])+ DiscourseDataExplorer::DefaultQuerySync.execute+ end+ end+describe "when disabled" dobefore { SiteSetting.data_explorer_enabled = false }@@ -62,6 +78,64 @@ describe DiscourseDataExplorer::QueryController doenddescribe "#index" do+ context "when default queries are synchronized" do+ include_context "with synchronized default queries"++ fab!(:custom_queries) { Fabricate.times(4, :query) }++ let(:query_parameters) { { order: "name", ascending: "true" } }+ let(:query_ids) do+ DiscourseDataExplorer::Query.where(hidden: false).order(:name).pluck(:id)+ end+ let(:first_page) { response_json }+ let(:second_page) { response_json }+ let(:first_page_ids) { first_page.fetch("queries").map { it["id"] } }+ let(:second_page_ids) { second_page.fetch("queries").map { it["id"] } }++ before do+ get "/admin/plugins/discourse-data-explorer/queries.json", params: query_parameters+ end++ it "paginates default and custom queries together", :aggregate_failures do+ expect(first_page_ids.size).to eq(50)+ expect(first_page.fetch("total_rows_queries")).to eq(query_ids.size)++ get first_page.fetch("load_more_queries")++ expect(first_page_ids + second_page_ids).to eq(query_ids)+ expect(second_page).not_to have_key("load_more_queries")+ end++ context "when the request filters by automatic and local tags" do+ let(:query_parameters) { super().merge(tags: "default,staff") }++ it "returns the matching default query" do+ expect(first_page_ids).to contain_exactly(existing_query.id)+ end+ end++ context "when queries use the default order" do+ let(:query_parameters) { super().except(:order, :ascending) }+ let(:query_ids) do+ DiscourseDataExplorer::Query.order(last_run_at: :desc, id: :asc).pluck(:id)+ end++ it "paginates queries with the same last run time" do+ get first_page.fetch("load_more_queries")++ expect(first_page_ids + second_page_ids).to eq(query_ids)+ end+ end++ context "when a default query is hidden" do+ let(:hidden) { true }++ it "excludes the query from the list" do+ expect(first_page_ids).not_to include(existing_query.id)+ end+ end+ end+it "behaves nicely with no user created queries" doDiscourseDataExplorer::Query.destroy_allget "/admin/plugins/discourse-data-explorer/queries.json"@@ -409,6 +483,31 @@ describe DiscourseDataExplorer::QueryController doenddescribe "#show" do+ context "when default queries are synchronized" do+ include_context "with synchronized default queries"++ before { get "/admin/plugins/discourse-data-explorer/queries/#{existing_query.id}.json" }++ it "serves the current definition with local query state" do+ expect(response_json.fetch("query")).to include(+ "id" => existing_query.id,+ "name" => definition.fetch("name"),+ "sql" => definition.fetch("sql"),+ "group_ids" => [group.id],+ "last_run_at" => existing_query.last_run_at.as_json,+ "tags" => %w[default users staff],+ )+ end++ context "when a default query is hidden" do+ let(:hidden) { true }++ it "returns not found for the query" do+ expect(response).to have_http_status(:not_found)+ end+ end+ end+it "reports whether the query can be, and is, on the admin dashboard" doquery = make_query("SELECT 1 as value")@@ -488,6 +587,17 @@ describe DiscourseDataExplorer::QueryController doenddescribe "#run" do+ context "when default queries are synchronized" do+ include_context "with synchronized default queries"++ it "runs a persisted default query", :aggregate_failures do+ run_query existing_query.id++ expect(response).to have_http_status(:ok)+ expect(existing_query.reload.last_run_at).to be_within(1.second).of(Time.current)+ end+ end+it "can run queries" doquery = make_query("SELECT 23 as my_value")run_query query.id