Skip to content

A platform for community discussion. Free, open, simple.

PERF: add pagination to data explorer queries list (#38547)

Rafael dos Santos Silvaf4b58d5e1166
diff --git a/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/index.gjs b/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/index.gjs
index d57ace885c1..ad3281eeb7d 100644
--- a/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/index.gjs
+++ b/plugins/discourse-data-explorer/admin/assets/javascripts/admin/templates/admin-plugins/show/explorer/index.gjs
@@ -1,10 +1,11 @@
-import { array, fn } from "@ember/helper";
+import { fn } from "@ember/helper";
import { on } from "@ember/modifier";
import { LinkTo } from "@ember/routing";
import AdminFilterControls from "discourse/admin/components/admin-filter-controls";
import ConditionalLoadingSpinner from "discourse/components/conditional-loading-spinner";
import DPageSubheader from "discourse/components/d-page-subheader";
import Form from "discourse/components/form";
+import LoadMore from "discourse/components/load-more";
import PickFilesButton from "discourse/components/pick-files-button";
import TableHeaderToggle from "discourse/components/table-header-toggle";
import ageWithTooltip from "discourse/helpers/age-with-tooltip";
@@ -39,10 +40,12 @@ export default <template>
</DPageSubheader>
<AdminFilterControls
- @array={{@controller.sortedQueries}}
- @searchableProps={{array "name" "description"}}
+ @array={{@controller.model.content}}
@inputPlaceholder={{i18n "explorer.search_placeholder"}}
@noResultsMessage={{i18n "explorer.no_search_results"}}
+ @onTextFilterChange={{@controller.onTextFilterChange}}
+ @onResetFilters={{@controller.onResetFilters}}
+ @loading={{@controller.searchLoading}}
>
<:aboveFilters>
{{#if @controller.showCreate}}
@@ -86,154 +89,169 @@ export default <template>
{{#if @controller.model.content.length}}
<ConditionalLoadingSpinner @condition={{@controller.loading}} />
- <div class="container discourse-data-explorer-query-list">
- <table class="d-table recent-queries">
- <thead class="d-table__header heading-container">
- <th class="col heading name">
- <div
- role="button"
- class="heading-toggle"
- {{on "click" (fn @controller.updateSortProperty "name")}}
- >
- <TableHeaderToggle
- @field="name"
- @labelKey="explorer.query_name"
- @order={{@controller.order}}
- @asc={{not @controller.sortDescending}}
- @automatic="true"
- />
- </div>
- </th>
- <th class="col heading created-by">
- <div
- role="button"
- class="heading-toggle"
- {{on
- "click"
- (fn @controller.updateSortProperty "username")
- }}
- >
- <TableHeaderToggle
- @field="username"
- @labelKey="explorer.query_user"
- @order={{@controller.order}}
- @asc={{not @controller.sortDescending}}
- @automatic="true"
- />
- </div>
- </th>
- <th class="col heading group-names">
- <div class="group-names-header">
- {{i18n "explorer.query_groups"}}
- </div>
- </th>
- <th class="col heading created-at">
- <div
- role="button"
- class="heading-toggle"
- {{on
- "click"
- (fn @controller.updateSortProperty "last_run_at")
- }}
- >
- <TableHeaderToggle
- @field="last_run_at"
- @labelKey="explorer.query_time"
- @order={{@controller.order}}
- @asc={{not @controller.sortDescending}}
- @automatic="true"
- />
- </div>
- </th>
- </thead>
- <tbody>
- {{#each filteredQueries as |query|}}
- <tr class="d-table__row query-row">
- <td class="d-table__cell --overview">
- <LinkTo
- class="d-table__overview-link"
- @route="adminPlugins.show.explorer.details"
- @model={{query.id}}
- >
- <div
- class="d-table__overview-name query-name"
- >{{query.name}}
- {{#if query.is_default}}
- <span class="query-badge">{{i18n
- "explorer.default_query"
- }}</span>
- {{/if}}
- </div>
- <div class="query-desc">{{query.description}}</div>
- </LinkTo>
- </td>
- <td class="d-table__cell --detail query-created-by">
- <div class="d-table__mobile-label">
- {{i18n "explorer.query_user"}}
- </div>
- {{#if query.username}}
- <div>
- <LinkTo
- @route="user.summary"
- @model={{query.username}}
- >
- {{query.username}}
- </LinkTo>
- </div>
- {{/if}}
- </td>
- <td class="d-table__cell --detail query-group-names">
- <div class="d-table__mobile-label">
- {{i18n "explorer.query_groups"}}
- </div>
- <div class="group-names">
- {{#each query.group_names as |group|}}
- <ShareReport @group={{group}} @query={{query}} />
- {{/each}}
- {{#unless query.group_names.length}}
- -
- {{/unless}}
- </div>
- </td>
- <td class="d-table__cell --detail query-created-at">
- <div class="d-table__mobile-label">
- {{i18n "explorer.query_time"}}
- </div>
- {{#if query.last_run_at}}
- <span>
- {{ageWithTooltip query.last_run_at format="medium"}}
- </span>
- {{else if query.created_at}}
- <span>
- {{ageWithTooltip query.created_at format="medium"}}
- </span>
- {{else}}
- -
- {{/if}}
- </td>
- <td class="d-table__cell-controls">
- <div class="d-table__cell-actions">
+ <LoadMore @action={{@controller.loadMore}}>
+ <div class="container discourse-data-explorer-query-list">
+ <table class="d-table recent-queries">
+ <thead class="d-table__header heading-container">
+ <th class="col heading name">
+ <div
+ role="button"
+ class="heading-toggle"
+ {{on
+ "click"
+ (fn @controller.updateSortProperty "name")
+ }}
+ >
+ <TableHeaderToggle
+ @field="name"
+ @labelKey="explorer.query_name"
+ @order={{@controller.order}}
+ @asc={{not @controller.sortDescending}}
+ @automatic="true"
+ />
+ </div>
+ </th>
+ <th class="col heading created-by">
+ <div
+ role="button"
+ class="heading-toggle"
+ {{on
+ "click"
+ (fn @controller.updateSortProperty "username")
+ }}
+ >
+ <TableHeaderToggle
+ @field="username"
+ @labelKey="explorer.query_user"
+ @order={{@controller.order}}
+ @asc={{not @controller.sortDescending}}
+ @automatic="true"
+ />
+ </div>
+ </th>
+ <th class="col heading group-names">
+ <div class="group-names-header">
+ {{i18n "explorer.query_groups"}}
+ </div>
+ </th>
+ <th class="col heading created-at">
+ <div
+ role="button"
+ class="heading-toggle"
+ {{on
+ "click"
+ (fn @controller.updateSortProperty "last_run_at")
+ }}
+ >
+ <TableHeaderToggle
+ @field="last_run_at"
+ @labelKey="explorer.query_time"
+ @order={{@controller.order}}
+ @asc={{not @controller.sortDescending}}
+ @automatic="true"
+ />
+ </div>
+ </th>
+ </thead>
+ <tbody>
+ {{#each filteredQueries as |query|}}
+ <tr class="d-table__row query-row">
+ <td class="d-table__cell --overview">
<LinkTo
- {{on "click" @controller.scrollTop}}
+ class="d-table__overview-link"
@route="adminPlugins.show.explorer.details"
@model={{query.id}}
- class="btn btn-default btn-small"
>
- {{i18n "edit"}}
+ <div
+ class="d-table__overview-name query-name"
+ >{{query.name}}
+ {{#if query.is_default}}
+ <span class="query-badge">{{i18n
+ "explorer.default_query"
+ }}</span>
+ {{/if}}
+ </div>
+ <div class="query-desc">{{query.description}}</div>
</LinkTo>
- </div>
+ </td>
+ <td class="d-table__cell --detail query-created-by">
+ <div class="d-table__mobile-label">
+ {{i18n "explorer.query_user"}}
+ </div>
+ {{#if query.username}}
+ <div>
+ <LinkTo
+ @route="user.summary"
+ @model={{query.username}}
+ >
+ {{query.username}}
+ </LinkTo>
+ </div>
+ {{/if}}
+ </td>
+ <td class="d-table__cell --detail query-group-names">
+ <div class="d-table__mobile-label">
+ {{i18n "explorer.query_groups"}}
+ </div>
+ <div class="group-names">
+ {{#each query.group_names as |group|}}
+ <ShareReport @group={{group}} @query={{query}} />
+ {{/each}}
+ {{#unless query.group_names.length}}
+ -
+ {{/unless}}
+ </div>
+ </td>
+ <td class="d-table__cell --detail query-created-at">
+ <div class="d-table__mobile-label">
+ {{i18n "explorer.query_time"}}
+ </div>
+ {{#if query.last_run_at}}
+ <span>
+ {{ageWithTooltip
+ query.last_run_at
+ format="medium"
+ }}
+ </span>
+ {{else if query.created_at}}
+ <span>
+ {{ageWithTooltip
+ query.created_at
+ format="medium"
+ }}
+ </span>
+ {{else}}
+ -
+ {{/if}}
+ </td>
+ <td class="d-table__cell-controls">
+ <div class="d-table__cell-actions">
+ <LinkTo
+ {{on "click" @controller.scrollTop}}
+ @route="adminPlugins.show.explorer.details"
+ @model={{query.id}}
+ class="btn btn-default btn-small"
+ >
+ {{i18n "edit"}}
+ </LinkTo>
+ </div>
- </td>
- </tr>
- {{else}}
- <br />
- <em class="no-search-results">
- {{i18n "explorer.no_search_results"}}
- </em>
- {{/each}}
- </tbody>
- </table>
- </div>
+ </td>
+ </tr>
+ {{else}}
+ <br />
+ <em class="no-search-results">
+ {{i18n "explorer.no_search_results"}}
+ </em>
+ {{/each}}
+ </tbody>
+ </table>
+ </div>
+ </LoadMore>
+
+ <ConditionalLoadingSpinner
+ @condition={{@controller.model.loadingMore}}
+ />
<div class="explorer-pad-bottom"></div>
{{/if}}
diff --git a/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb b/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb
index 118c1227638..dd5fd59c9e1 100644
--- a/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb
+++ b/plugins/discourse-data-explorer/app/controllers/discourse_data_explorer/query_controller.rb
@@ -12,31 +12,86 @@ module DiscourseDataExplorer
skip_before_action :ensure_admin,
only: %i[group_reports_index group_reports_show group_reports_run public_run]
+ INDEX_LIMIT = 50
+
+ SORTABLE_COLUMNS = %w[name username last_run_at].freeze
+
def index
- queries =
- DiscourseDataExplorer::Query
- .where(hidden: false)
- .order(:last_run_at, :name)
- .includes(:groups)
- .to_a
-
- DiscourseDataExplorer::Queries.default.each do |params|
- attributes = params.last
- persisted = queries.find { |q| q.id == attributes["id"] }
-
- # Once a default query has been run, it is also saved in the local db
- # so we can skip creating a "fake" query for the UI here, and this also
- # avoids double-ups of the default query in the UI.
- next if persisted.present?
-
- query =
- DiscourseDataExplorer::Query.new(attributes.slice("id", "sql", "name", "description"))
- query.user_id = Discourse::SYSTEM_USER_ID.to_s
-
- queries << query
+ limit = INDEX_LIMIT
+ offset = params[:offset].to_i
+ filter = params[:filter]
+
+ order_column = SORTABLE_COLUMNS.include?(params[:order]) ? params[:order] : "last_run_at"
+ order_direction = params[:ascending] == "true" ? :asc : :desc
+
+ base_scope = DiscourseDataExplorer::Query.where(hidden: false).includes(:groups)
+
+ if order_column == "username"
+ base_scope =
+ base_scope.joins("LEFT JOIN users ON users.id = data_explorer_queries.user_id").order(
+ Arel.sql("users.username #{order_direction}"),
+ )
+ else
+ base_scope = base_scope.order(order_column => order_direction)
+ end
+
+ if filter.present?
+ sanitized_filter = "%#{Query.sanitize_sql_like(filter)}%"
+ base_scope =
+ base_scope.where(
+ "data_explorer_queries.name ILIKE :filter OR data_explorer_queries.description ILIKE :filter",
+ filter: sanitized_filter,
+ )
+ end
+
+ persisted_count = base_scope.count
+
+ # Default queries are only persisted once run. Build in-memory records
+ # for any that haven't been run yet so they still appear in the list.
+ persisted_default_ids =
+ DiscourseDataExplorer::Query.where(hidden: false).where("id < 0").pluck(:id).to_set
+ unpersisted_defaults =
+ DiscourseDataExplorer::Queries.default.filter_map do |_, attributes|
+ next if persisted_default_ids.include?(attributes["id"])
+ if filter.present?
+ name_match = attributes["name"]&.downcase&.include?(filter.downcase)
+ desc_match = attributes["description"]&.downcase&.include?(filter.downcase)
+ next unless name_match || desc_match
+ end
+ query =
+ DiscourseDataExplorer::Query.new(attributes.slice("id", "sql", "name", "description"))
+ query.user_id = Discourse::SYSTEM_USER_ID.to_s
+ query
+ end
+
+ total_rows = persisted_count + unpersisted_defaults.size
+
+ # On the first page, fit defaults within the limit so the page size
+ # stays consistent. On subsequent pages, only DB results are returned.
+ if offset == 0
+ db_limit = [limit - unpersisted_defaults.size, 0].max
+ paginated = base_scope.limit(db_limit).to_a
+ queries = paginated + unpersisted_defaults
+ next_offset = paginated.size
+ else
+ paginated = base_scope.offset(offset).limit(limit).to_a
+ queries = paginated
+ next_offset = offset + limit
+ end
+
+ json = serialize_data(queries, QuerySerializer, root: "queries")
+ json["total_rows_queries"] = total_rows
+
+ if next_offset < persisted_count
+ load_more_params = { offset: next_offset }
+ load_more_params[:filter] = filter if filter.present?
+ load_more_params[:order] = order_column if order_column != "last_run_at"
+ load_more_params[:ascending] = "true" if order_direction == :asc
+ base_path = request.path.delete_suffix(".json")
+ json["load_more_queries"] = "#{base_path}.json?#{load_more_params.to_query}"
end
- render_serialized(queries, QuerySerializer, root: "queries")
+ render_json_dump(json)
end
def show
diff --git a/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/index.js b/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/index.js
index 354e6b4e89a..1557b1ba727 100644
--- a/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/index.js
+++ b/plugins/discourse-data-explorer/assets/javascripts/discourse/controllers/admin-plugins/show/explorer/index.js
@@ -2,16 +2,19 @@ import { tracked } from "@glimmer/tracking";
import Controller from "@ember/controller";
import { action } from "@ember/object";
import { service } from "@ember/service";
-import { compare } from "@ember/utils";
import { Promise } from "rsvp";
+import { ajax } from "discourse/lib/ajax";
import { popupAjaxError } from "discourse/lib/ajax-error";
+import discourseDebounce from "discourse/lib/debounce";
import { bind } from "discourse/lib/decorators";
+import { INPUT_DELAY } from "discourse/lib/environment";
import { i18n } from "discourse-i18n";
export default class PluginsExplorerController extends Controller {
@service dialog;
@service appEvents;
@service router;
+ @service store;
@service toasts;
@tracked sortByProperty = "last_run_at";
@@ -20,19 +23,14 @@ export default class PluginsExplorerController extends Controller {
@tracked createFormData = { name: "" };
@tracked showCreate;
@tracked loading = false;
+ @tracked searchLoading = false;
queryParams = ["id"];
explain = false;
acceptedImportFileTypes = ["application/json"];
order = null;
form = null;
-
- get sortedQueries() {
- const sortedQueries = this.model.content.toSorted((a, b) =>
- compare(a?.[this.sortByProperty], b?.[this.sortByProperty])
- );
- return this.sortDescending ? sortedQueries.reverse() : sortedQueries;
- }
+ _currentFilter = "";
get parsedParams() {
return this.params ? JSON.parse(this.params) : null;
@@ -145,6 +143,20 @@ export default class PluginsExplorerController extends Controller {
this.showCreate = false;
}
+ @action
+ onTextFilterChange(event) {
+ this._currentFilter = event.target?.value || "";
+ this.searchLoading = true;
+ discourseDebounce(this, this._fetchQueries, INPUT_DELAY);
+ }
+
+ @action
+ onResetFilters() {
+ this._currentFilter = "";
+ this.searchLoading = true;
+ this._fetchQueries();
+ }
+
@action
updateSortProperty(property) {
if (this.sortByProperty === property) {
@@ -153,6 +165,69 @@ export default class PluginsExplorerController extends Controller {
this.sortByProperty = property;
this.sortDescending = true;
}
+ this.searchLoading = true;
+ this._fetchQueries();
+ }
+
+ @action
+ async loadMore() {
+ const loadedMore = await this.model?.loadMore();
+ if (loadedMore) {
+ this._setGroupNames(this.model.content);
+ }
+ }
+
+ get _fetchParams() {
+ const params = {};
+ if (this._currentFilter) {
+ params.filter = this._currentFilter;
+ }
+ if (this.sortByProperty !== "last_run_at") {
+ params.order = this.sortByProperty;
+ }
+ if (!this.sortDescending) {
+ params.ascending = "true";
+ }
+ return params;
+ }
+
+ async _fetchQueries() {
+ try {
+ const result = await ajax(
+ "/admin/plugins/discourse-data-explorer/queries.json",
+ { data: this._fetchParams }
+ );
+
+ const queries = result.queries.map((q) =>
+ this.store.createRecord("query", q)
+ );
+
+ this.model.content.splice(0, this.model.content.length, ...queries);
+ this.model.totalRows = result.total_rows_queries || queries.length;
+ this.model.loadMoreUrl = result.load_more_queries || null;
+
+ this._setGroupNames(queries);
+ } catch (e) {
+ popupAjaxError(e);
+ } finally {
+ this.searchLoading = false;
+ }
+ }
+
+ _setGroupNames(queries) {
+ if (!this.groups) {
+ return;
+ }
+ const groupNames = {};
+ this.groups.forEach((g) => {
+ groupNames[g.id] = g.name;
+ });
+ queries.forEach((query) => {
+ query.set(
+ "group_names",
+ (query.group_ids || []).map((id) => groupNames[id])
+ );
+ });
}
@action
diff --git a/plugins/discourse-data-explorer/assets/stylesheets/explorer.scss b/plugins/discourse-data-explorer/assets/stylesheets/explorer.scss
index e31ee64614c..aeb9f7b7864 100644
--- a/plugins/discourse-data-explorer/assets/stylesheets/explorer.scss
+++ b/plugins/discourse-data-explorer/assets/stylesheets/explorer.scss
@@ -445,6 +445,7 @@ table.group-reports {
.query-name {
color: var(--primary);
+ max-width: 100%;
}
.query-badge {
diff --git a/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb b/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb
index 9a61fb98aa6..f2a7dbef1b5 100644
--- a/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb
+++ b/plugins/discourse-data-explorer/spec/requests/query_controller_spec.rb
@@ -59,11 +59,15 @@ describe DiscourseDataExplorer::QueryController do
expect(response_json["queries"].count).to eq(DiscourseDataExplorer::Queries.default.count)
end
- it "shows all available queries in alphabetical order" do
+ it "shows all available queries sorted by name when requested" do
DiscourseDataExplorer::Query.destroy_all
make_query("SELECT 1 as value", name: "B")
make_query("SELECT 1 as value", name: "A")
- get "/admin/plugins/discourse-data-explorer/queries.json"
+ get "/admin/plugins/discourse-data-explorer/queries.json",
+ params: {
+ order: "name",
+ ascending: "true",
+ }
expect(response.status).to eq(200)
expect(response_json["queries"].length).to eq(
DiscourseDataExplorer::Queries.default.count + 2,
diff --git a/plugins/discourse-data-explorer/test/javascripts/acceptance/list-queries-test.js b/plugins/discourse-data-explorer/test/javascripts/acceptance/list-queries-test.js
index f855e0e27aa..5f4f607130d 100644
--- a/plugins/discourse-data-explorer/test/javascripts/acceptance/list-queries-test.js
+++ b/plugins/discourse-data-explorer/test/javascripts/acceptance/list-queries-test.js
@@ -32,26 +32,27 @@ acceptance("Data Explorer Plugin | List Queries", function (needs) {
return helper.response({
queries: [
{
- id: -5,
- name: "Top 100 Active Topics",
+ id: -6,
+ name: "Top 100 Likers",
description:
- "based on the number of replies, it accepts a ‘months_ago’ parameter, defaults to 1 to give results for the last calendar month.",
+ "returns the top 100 likers for a given monthly period ordered by like_count. It accepts a ‘months_ago’ parameter, defaults to 1 to give results for the last calendar month.",
username: "system",
group_ids: [],
- last_run_at: "2021-02-08T15:37:49.188Z",
+ last_run_at: "2021-02-11T08:29:59.337Z",
user_id: -1,
},
{
- id: -6,
- name: "Top 100 Likers",
+ id: -5,
+ name: "Top 100 Active Topics",
description:
- "returns the top 100 likers for a given monthly period ordered by like_count. It accepts a ‘months_ago’ parameter, defaults to 1 to give results for the last calendar month.",
+ "based on the number of replies, it accepts a ‘months_ago’ parameter, defaults to 1 to give results for the last calendar month.",
username: "system",
group_ids: [],
- last_run_at: "2021-02-11T08:29:59.337Z",
+ last_run_at: "2021-02-08T15:37:49.188Z",
user_id: -1,
},
],
+ total_rows_queries: 2,
});
});
});