A platform for community discussion. Free, open, simple.
PERF: add pagination to data explorer queries list (#38547)
Rafael dos Santos Silvaf4b58d5e1166
diff --git a/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/index.gjs b/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/index.gjsindex d57ace885c1..ad3281eeb7d 100644--- a/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/index.gjs+++ b/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/index.gjs@@ -1,10 +1,11 @@-import { array, fn } from "@ember/helper";+import { fn } from "@ember/helper";import { on } from "@ember/modifier";import { LinkTo } from "@ember/routing";import AdminFilterControls from "discourse/admin/components/admin-filter-controls";import ConditionalLoadingSpinner from "discourse/components/conditional-loading-spinner";import DPageSubheader from "discourse/components/d-page-subheader";import Form from "discourse/components/form";+import LoadMore from "discourse/components/load-more";import PickFilesButton from "discourse/components/pick-files-button";import TableHeaderToggle from "discourse/components/table-header-toggle";import ageWithTooltip from "discourse/helpers/age-with-tooltip";@@ -39,10 +40,12 @@ export default <template></DPageSubheader><AdminFilterControls- @array={{@controller.sortedQueries}}- @searchableProps={{array "name" "description"}}+ @array={{@controller.model.content}}@inputPlaceholder={{i18n "explorer.search_placeholder"}}@noResultsMessage={{i18n "explorer.no_search_results"}}+ @onTextFilterChange={{@controller.onTextFilterChange}}+ @onResetFilters={{@controller.onResetFilters}}+ @loading={{@controller.searchLoading}}><:aboveFilters>{{#if @controller.showCreate}}@@ -86,154 +89,169 @@ export default <template>{{#if @controller.model.content.length}}<ConditionalLoadingSpinner @condition={{@controller.loading}} />- <div class="container discourse-data-explorer-query-list">- <table class="d-table recent-queries">- <thead class="d-table__header heading-container">- <th class="col heading name">- <div- role="button"- class="heading-toggle"- {{on "click" (fn @controller.updateSortProperty "name")}}- >- <TableHeaderToggle- @field="name"- @labelKey="explorer.query_name"- @order={{@controller.order}}- @asc={{not @controller.sortDescending}}- @automatic="true"- />- </div>- </th>- <th class="col heading created-by">- <div- role="button"- class="heading-toggle"- {{on- "click"- (fn @controller.updateSortProperty "username")- }}- >- <TableHeaderToggle- @field="username"- @labelKey="explorer.query_user"- @order={{@controller.order}}- @asc={{not @controller.sortDescending}}- @automatic="true"- />- </div>- </th>- <th class="col heading group-names">- <div class="group-names-header">- {{i18n "explorer.query_groups"}}- </div>- </th>- <th class="col heading created-at">- <div- role="button"- class="heading-toggle"- {{on- "click"- (fn @controller.updateSortProperty "last_run_at")- }}- >- <TableHeaderToggle- @field="last_run_at"- @labelKey="explorer.query_time"- @order={{@controller.order}}- @asc={{not @controller.sortDescending}}- @automatic="true"- />- </div>- </th>- </thead>- <tbody>- {{#each filteredQueries as |query|}}- <tr class="d-table__row query-row">- <td class="d-table__cell --overview">- <LinkTo- class="d-table__overview-link"- @route="adminPlugins.show.explorer.details"- @model={{query.id}}- >- <div- class="d-table__overview-name query-name"- >{{query.name}}- {{#if query.is_default}}- <span class="query-badge">{{i18n- "explorer.default_query"- }}</span>- {{/if}}- </div>- <div class="query-desc">{{query.description}}</div>- </LinkTo>- </td>- <td class="d-table__cell --detail query-created-by">- <div class="d-table__mobile-label">- {{i18n "explorer.query_user"}}- </div>- {{#if query.username}}- <div>- <LinkTo- @route="user.summary"- @model={{query.username}}- >- {{query.username}}- </LinkTo>- </div>- {{/if}}- </td>- <td class="d-table__cell --detail query-group-names">- <div class="d-table__mobile-label">- {{i18n "explorer.query_groups"}}- </div>- <div class="group-names">- {{#each query.group_names as |group|}}- <ShareReport @group={{group}} @query={{query}} />- {{/each}}- {{#unless query.group_names.length}}- -- {{/unless}}- </div>- </td>- <td class="d-table__cell --detail query-created-at">- <div class="d-table__mobile-label">- {{i18n "explorer.query_time"}}- </div>- {{#if query.last_run_at}}- <span>- {{ageWithTooltip query.last_run_at format="medium"}}- </span>- {{else if query.created_at}}- <span>- {{ageWithTooltip query.created_at format="medium"}}- </span>- {{else}}- -- {{/if}}- </td>- <td class="d-table__cell-controls">- <div class="d-table__cell-actions">+ <LoadMore @action={{@controller.loadMore}}>+ <div class="container discourse-data-explorer-query-list">+ <table class="d-table recent-queries">+ <thead class="d-table__header heading-container">+ <th class="col heading name">+ <div+ role="button"+ class="heading-toggle"+ {{on+ "click"+ (fn @controller.updateSortProperty "name")+ }}+ >+ <TableHeaderToggle+ @field="name"+ @labelKey="explorer.query_name"+ @order={{@controller.order}}+ @asc={{not @controller.sortDescending}}+ @automatic="true"+ />+ </div>+ </th>+ <th class="col heading created-by">+ <div+ role="button"+ class="heading-toggle"+ {{on+ "click"+ (fn @controller.updateSortProperty "username")+ }}+ >+ <TableHeaderToggle+ @field="username"+ @labelKey="explorer.query_user"+ @order={{@controller.order}}+ @asc={{not @controller.sortDescending}}+ @automatic="true"+ />+ </div>+ </th>+ <th class="col heading group-names">+ <div class="group-names-header">+ {{i18n "explorer.query_groups"}}+ </div>+ </th>+ <th class="col heading created-at">+ <div+ role="button"+ class="heading-toggle"+ {{on+ "click"+ (fn @controller.updateSortProperty "last_run_at")+ }}+ >+ <TableHeaderToggle+ @field="last_run_at"+ @labelKey="explorer.query_time"+ @order={{@controller.order}}+ @asc={{not @controller.sortDescending}}+ @automatic="true"+ />+ </div>+ </th>+ </thead>+ <tbody>+ {{#each filteredQueries as |query|}}+ <tr class="d-table__row query-row">+ <td class="d-table__cell --overview"><LinkTo- {{on "click" @controller.scrollTop}}+ class="d-table__overview-link"@route="adminPlugins.show.explorer.details"@model={{query.id}}- class="btn btn-default btn-small">- {{i18n "edit"}}+ <div+ class="d-table__overview-name query-name"+ >{{query.name}}+ {{#if query.is_default}}+ <span class="query-badge">{{i18n+ "explorer.default_query"+ }}</span>+ {{/if}}+ </div>+ <div class="query-desc">{{query.description}}</div></LinkTo>- </div>+ </td>+ <td class="d-table__cell --detail query-created-by">+ <div class="d-table__mobile-label">+ {{i18n "explorer.query_user"}}+ </div>+ {{#if query.username}}+ <div>+ <LinkTo+ @route="user.summary"+ @model={{query.username}}+ >+ {{query.username}}+ </LinkTo>+ </div>+ {{/if}}+ </td>+ <td class="d-table__cell --detail query-group-names">+ <div class="d-table__mobile-label">+ {{i18n "explorer.query_groups"}}+ </div>+ <div class="group-names">+ {{#each query.group_names as |group|}}+ <ShareReport @group={{group}} @query={{query}} />+ {{/each}}+ {{#unless query.group_names.length}}+ -+ {{/unless}}+ </div>+ </td>+ <td class="d-table__cell --detail query-created-at">+ <div class="d-table__mobile-label">+ {{i18n "explorer.query_time"}}+ </div>+ {{#if query.last_run_at}}+ <span>+ {{ageWithTooltip+ query.last_run_at+ format="medium"+ }}+ </span>+ {{else if query.created_at}}+ <span>+ {{ageWithTooltip+ query.created_at+ format="medium"+ }}+ </span>+ {{else}}+ -+ {{/if}}+ </td>+ <td class="d-table__cell-controls">+ <div class="d-table__cell-actions">+ <LinkTo+ {{on "click" @controller.scrollTop}}+ @route="adminPlugins.show.explorer.details"+ @model={{query.id}}+ class="btn btn-default btn-small"+ >+ {{i18n "edit"}}+ </LinkTo>+ </div>- </td>- </tr>- {{else}}- <br />- <em class="no-search-results">- {{i18n "explorer.no_search_results"}}- </em>- {{/each}}- </tbody>- </table>- </div>+ </td>+ </tr>+ {{else}}+ <br />+ <em class="no-search-results">+ {{i18n "explorer.no_search_results"}}+ </em>+ {{/each}}+ </tbody>+ </table>+ </div>+ </LoadMore>++ <ConditionalLoadingSpinner+ @condition={{@controller.model.loadingMore}}+ /><div class="explorer-pad-bottom"></div>{{/if}}diff --git a/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb b/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rbindex 118c1227638..dd5fd59c9e1 100644--- a/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb+++ b/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb@@ -12,31 +12,86 @@ module DiscourseDataExplorerskip_before_action :ensure_admin,only: %i[group_reports_index group_reports_show group_reports_run public_run]+ INDEX_LIMIT = 50++ SORTABLE_COLUMNS = %w[name username last_run_at].freeze+def index- queries =- DiscourseDataExplorer::Query- .where(hidden: false)- .order(:last_run_at, :name)- .includes(:groups)- .to_a-- DiscourseDataExplorer::Queries.default.each do |params|- attributes = params.last- persisted = queries.find { |q| q.id == attributes["id"] }-- # Once a default query has been run, it is also saved in the local db- # so we can skip creating a "fake" query for the UI here, and this also- # avoids double-ups of the default query in the UI.- next if persisted.present?-- query =- DiscourseDataExplorer::Query.new(attributes.slice("id", "sql", "name", "description"))- query.user_id = Discourse::SYSTEM_USER_ID.to_s-- queries << query+ limit = INDEX_LIMIT+ offset = params[:offset].to_i+ filter = params[:filter]++ order_column = SORTABLE_COLUMNS.include?(params[:order]) ? params[:order] : "last_run_at"+ order_direction = params[:ascending] == "true" ? :asc : :desc++ base_scope = DiscourseDataExplorer::Query.where(hidden: false).includes(:groups)++ if order_column == "username"+ base_scope =+ base_scope.joins("LEFT JOIN users ON users.id = data_explorer_queries.user_id").order(+ Arel.sql("users.username #{order_direction}"),+ )+ else+ base_scope = base_scope.order(order_column => order_direction)+ end++ if filter.present?+ sanitized_filter = "%#{Query.sanitize_sql_like(filter)}%"+ base_scope =+ base_scope.where(+ "data_explorer_queries.name ILIKE :filter OR data_explorer_queries.description ILIKE :filter",+ filter: sanitized_filter,+ )+ end++ persisted_count = base_scope.count++ # Default queries are only persisted once run. Build in-memory records+ # for any that haven't been run yet so they still appear in the list.+ persisted_default_ids =+ DiscourseDataExplorer::Query.where(hidden: false).where("id < 0").pluck(:id).to_set+ unpersisted_defaults =+ DiscourseDataExplorer::Queries.default.filter_map do |_, attributes|+ next if persisted_default_ids.include?(attributes["id"])+ if filter.present?+ name_match = attributes["name"]&.downcase&.include?(filter.downcase)+ desc_match = attributes["description"]&.downcase&.include?(filter.downcase)+ next unless name_match || desc_match+ end+ query =+ DiscourseDataExplorer::Query.new(attributes.slice("id", "sql", "name", "description"))+ query.user_id = Discourse::SYSTEM_USER_ID.to_s+ query+ end++ total_rows = persisted_count + unpersisted_defaults.size++ # On the first page, fit defaults within the limit so the page size+ # stays consistent. On subsequent pages, only DB results are returned.+ if offset == 0+ db_limit = [limit - unpersisted_defaults.size, 0].max+ paginated = base_scope.limit(db_limit).to_a+ queries = paginated + unpersisted_defaults+ next_offset = paginated.size+ else+ paginated = base_scope.offset(offset).limit(limit).to_a+ queries = paginated+ next_offset = offset + limit+ end++ json = serialize_data(queries, QuerySerializer, root: "queries")+ json["total_rows_queries"] = total_rows++ if next_offset < persisted_count+ load_more_params = { offset: next_offset }+ load_more_params[:filter] = filter if filter.present?+ load_more_params[:order] = order_column if order_column != "last_run_at"+ load_more_params[:ascending] = "true" if order_direction == :asc+ base_path = request.path.delete_suffix(".json")+ json["load_more_queries"] = "#{base_path}.json?#{load_more_params.to_query}"end- render_serialized(queries, QuerySerializer, root: "queries")+ render_json_dump(json)enddef showdiff --git a/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/index.js b/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/index.jsindex 354e6b4e89a..1557b1ba727 100644--- a/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/index.js+++ b/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/index.js@@ -2,16 +2,19 @@ import { tracked } from "@glimmer/tracking";import Controller from "@ember/controller";import { action } from "@ember/object";import { service } from "@ember/service";-import { compare } from "@ember/utils";import { Promise } from "rsvp";+import { ajax } from "discourse/lib/ajax";import { popupAjaxError } from "discourse/lib/ajax-error";+import discourseDebounce from "discourse/lib/debounce";import { bind } from "discourse/lib/decorators";+import { INPUT_DELAY } from "discourse/lib/environment";import { i18n } from "discourse-i18n";export default class PluginsExplorerController extends Controller {@service dialog;@service appEvents;@service router;+ @service store;@service toasts;@tracked sortByProperty = "last_run_at";@@ -20,19 +23,14 @@ export default class PluginsExplorerController extends Controller {@tracked createFormData = { name: "" };@tracked showCreate;@tracked loading = false;+ @tracked searchLoading = false;queryParams = ["id"];explain = false;acceptedImportFileTypes = ["application/json"];order = null;form = null;-- get sortedQueries() {- const sortedQueries = this.model.content.toSorted((a, b) =>- compare(a?.[this.sortByProperty], b?.[this.sortByProperty])- );- return this.sortDescending ? sortedQueries.reverse() : sortedQueries;- }+ _currentFilter = "";get parsedParams() {return this.params ? JSON.parse(this.params) : null;@@ -145,6 +143,20 @@ export default class PluginsExplorerController extends Controller {this.showCreate = false;}+ @action+ onTextFilterChange(event) {+ this._currentFilter = event.target?.value || "";+ this.searchLoading = true;+ discourseDebounce(this, this._fetchQueries, INPUT_DELAY);+ }++ @action+ onResetFilters() {+ this._currentFilter = "";+ this.searchLoading = true;+ this._fetchQueries();+ }+@actionupdateSortProperty(property) {if (this.sortByProperty === property) {@@ -153,6 +165,69 @@ export default class PluginsExplorerController extends Controller {this.sortByProperty = property;this.sortDescending = true;}+ this.searchLoading = true;+ this._fetchQueries();+ }++ @action+ async loadMore() {+ const loadedMore = await this.model?.loadMore();+ if (loadedMore) {+ this._setGroupNames(this.model.content);+ }+ }++ get _fetchParams() {+ const params = {};+ if (this._currentFilter) {+ params.filter = this._currentFilter;+ }+ if (this.sortByProperty !== "last_run_at") {+ params.order = this.sortByProperty;+ }+ if (!this.sortDescending) {+ params.ascending = "true";+ }+ return params;+ }++ async _fetchQueries() {+ try {+ const result = await ajax(+ "/admin/plugins/discourse-data-explorer/queries.json",+ { data: this._fetchParams }+ );++ const queries = result.queries.map((q) =>+ this.store.createRecord("query", q)+ );++ this.model.content.splice(0, this.model.content.length, ...queries);+ this.model.totalRows = result.total_rows_queries || queries.length;+ this.model.loadMoreUrl = result.load_more_queries || null;++ this._setGroupNames(queries);+ } catch (e) {+ popupAjaxError(e);+ } finally {+ this.searchLoading = false;+ }+ }++ _setGroupNames(queries) {+ if (!this.groups) {+ return;+ }+ const groupNames = {};+ this.groups.forEach((g) => {+ groupNames[g.id] = g.name;+ });+ queries.forEach((query) => {+ query.set(+ "group_names",+ (query.group_ids || []).map((id) => groupNames[id])+ );+ });}@actiondiff --git a/plugins/discourse-data-explorer/assets/stylesheets/explorer.scss b/plugins/discourse-data-explorer/assets/stylesheets/explorer.scssindex e31ee64614c..aeb9f7b7864 100644--- a/plugins/discourse-data-explorer/assets/stylesheets/explorer.scss+++ b/plugins/discourse-data-explorer/assets/stylesheets/explorer.scss@@ -445,6 +445,7 @@ table.group-reports {.query-name {color: var(--primary);+ max-width: 100%;}.query-badge {diff --git a/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb b/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rbindex 9a61fb98aa6..f2a7dbef1b5 100644--- a/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb+++ b/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb@@ -59,11 +59,15 @@ describe DiscourseDataExplorer::QueryController doexpect(response_json["queries"].count).to eq(DiscourseDataExplorer::Queries.default.count)end- it "shows all available queries in alphabetical order" do+ it "shows all available queries sorted by name when requested" doDiscourseDataExplorer::Query.destroy_allmake_query("SELECT 1 as value", name: "B")make_query("SELECT 1 as value", name: "A")- get "/admin/plugins/discourse-data-explorer/queries.json"+ get "/admin/plugins/discourse-data-explorer/queries.json",+ params: {+ order: "name",+ ascending: "true",+ }expect(response.status).to eq(200)expect(response_json["queries"].length).to eq(DiscourseDataExplorer::Queries.default.count + 2,diff --git a/plugins/discourse-data-explorer/test/javascripts/acceptance/list-queries-test.js b/plugins/discourse-data-explorer/test/javascripts/acceptance/list-queries-test.jsindex f855e0e27aa..5f4f607130d 100644--- a/plugins/discourse-data-explorer/test/javascripts/acceptance/list-queries-test.js+++ b/plugins/discourse-data-explorer/test/javascripts/acceptance/list-queries-test.js@@ -32,26 +32,27 @@ acceptance("Data Explorer Plugin | List Queries", function (needs) {return helper.response({queries: [{- id: -5,- name: "Top 100 Active Topics",+ id: -6,+ name: "Top 100 Likers",description:- "based on the number of replies, it accepts a ‘months_ago’ parameter, defaults to 1 to give results for the last calendar month.",+ "returns the top 100 likers for a given monthly period ordered by like_count. It accepts a ‘months_ago’ parameter, defaults to 1 to give results for the last calendar month.",username: "system",group_ids: [],- last_run_at: "2021-02-08T15:37:49.188Z",+ last_run_at: "2021-02-11T08:29:59.337Z",user_id: -1,},{- id: -6,- name: "Top 100 Likers",+ id: -5,+ name: "Top 100 Active Topics",description:- "returns the top 100 likers for a given monthly period ordered by like_count. It accepts a ‘months_ago’ parameter, defaults to 1 to give results for the last calendar month.",+ "based on the number of replies, it accepts a ‘months_ago’ parameter, defaults to 1 to give results for the last calendar month.",username: "system",group_ids: [],- last_run_at: "2021-02-11T08:29:59.337Z",+ last_run_at: "2021-02-08T15:37:49.188Z",user_id: -1,},],+ total_rows_queries: 2,});});});