A platform for community discussion. Free, open, simple.
UX: Interleave dashboard report providers alphabetically (#40483)
Osama Sayegh50d99d9e4d3e
diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rbindex 0df17f60cf7..0c8f79bb588 100644--- a/app/controllers/admin/dashboard_controller.rb+++ b/app/controllers/admin/dashboard_controller.rb@@ -105,8 +105,9 @@ class Admin::DashboardController < Admin::StaffControllerdef available_reportssearch = params[:search]+ cursor = params.permit(cursor: %i[title key])[:cursor]&.to_h&.symbolize_keysenabled = AdminDashboard::Reports::Section.build(guardian: guardian, search: search)[:items]- listing = AdminDashboard::Reports::Listing.call(cursor: params[:cursor], search: search)+ listing = AdminDashboard::Reports::Listing.call(cursor: cursor, search: search)render json: {providers: listing[:providers],diff --git a/lib/admin_dashboard/reports/core_report_provider.rb b/lib/admin_dashboard/reports/core_report_provider.rbindex f8310a780a8..4fdc9384bce 100644--- a/lib/admin_dashboard/reports/core_report_provider.rb+++ b/lib/admin_dashboard/reports/core_report_provider.rb@@ -55,11 +55,10 @@ module AdminDashboardendprivate_class_method :with_empty_flag- def self.list_all(search: nil, offset: 0, limit: nil)+ def self.list_all(search: nil, after: nil, limit: nil)entries = ::Reports::ListQuery.call(guardian: Guardian.new(Discourse.system_user))entries = filter_by_search(entries, search) if search.present?- sliced = limit ? entries[offset, limit] : entries[offset..]- Array(sliced).map { |entry| build_resolved(entry) }+ seek(entries.map { |entry| build_resolved(entry) }, after: after, limit: limit)enddef self.build_resolved(entry)diff --git a/lib/admin_dashboard/reports/listing.rb b/lib/admin_dashboard/reports/listing.rbindex d7fd5ba101c..d13e0a1200a 100644--- a/lib/admin_dashboard/reports/listing.rb+++ b/lib/admin_dashboard/reports/listing.rb@@ -10,64 +10,50 @@ module AdminDashboardenddef initialize(cursor:, search:)- @cursor = parse_cursor(cursor)+ @cursor = normalize_cursor(cursor)@search = search.presenceenddef call- collected = walk_providers- has_more = collected.size > PAGE_SIZE+ merged =+ Registry+ .providers+ .flat_map do |provider|+ provider.list_all(search: @search, after: @cursor, limit: PAGE_SIZE + 1)+ end+ .sort_by { |report| [report.title.to_s.downcase, report.key] }++ has_more = merged.size > PAGE_SIZE+ page = merged.first(PAGE_SIZE){providers: provider_summaries,- items: collected.first(PAGE_SIZE).map { |entry| entry[:item].to_h },+ items: page.map(&:to_h),has_more: has_more,- cursor: has_more ? format_cursor(collected[PAGE_SIZE]) : nil,+ cursor: has_more ? cursor_for(page.last) : nil,}endprivate- def walk_providers- providers = Registry.providers- start_index = @cursor ? providers.find_index { |p| p.source_name == @cursor[:source] } : 0- walk = start_index ? providers[start_index..] : []- current_offset = @cursor ? @cursor[:offset] : 0-- collected = []- to_take = PAGE_SIZE + 1- walk.each do |provider|- break if to_take <= 0-- items = provider.list_all(search: @search, offset: current_offset, limit: to_take)- items.each_with_index do |item, i|- collected << { source: provider.source_name, item: item, offset: current_offset + i }- end- to_take -= items.size- current_offset = 0- end-- collected- end-def provider_summaries- Registry.providers.map { |p| { source: p.source_name, label: p.label } }+ Registry.providers.map do |provider|+ { source: provider.source_name, label: provider.label }+ endend- def parse_cursor(raw)+ def normalize_cursor(raw)return nil if raw.blank?- source, offset = raw.to_s.split(":", 2)- return nil if source.blank? || offset.blank?-- offset_int = Integer(offset, 10, exception: false)- return nil if offset_int.nil? || offset_int < 0+ title = raw[:title]+ key = raw[:key]+ return nil if title.blank? || key.blank?- { source: source, offset: offset_int }+ { title: title.to_s, key: key.to_s }end- def format_cursor(entry)- "#{entry[:source]}:#{entry[:offset]}"+ def cursor_for(report)+ { title: report.title, key: report.key }endendenddiff --git a/lib/admin_dashboard/reports/source_provider.rb b/lib/admin_dashboard/reports/source_provider.rbindex 99bed8ef8d5..b8abb49a578 100644--- a/lib/admin_dashboard/reports/source_provider.rb+++ b/lib/admin_dashboard/reports/source_provider.rb@@ -51,17 +51,39 @@ module AdminDashboard# Universe of items of this source. Powers the Manage Reports modal's# available list and search filter. Only invoked in admin-only contexts,# so implementations should not perform per-user access filtering here.- # Providers paginate themselves via `offset` and `limit` arguments; the- # controller orchestrates cross-provider pagination.+ #+ # Implements keyset pagination so the controller can merge every+ # provider into one globally title-sorted stream without loading the+ # whole universe. Items must come back sorted by+ # [title.downcase, key] (key being "source:identifier") and limited to+ # those strictly after `after`.## @param search [String, nil] optional name/description filter.- # @param offset [Integer] starting position within this provider's set.+ # @param after [Hash, nil] cursor of the last item from the previous+ # page: { title:, key: }. nil for the first page.# @param limit [Integer, nil] maximum number of items to return.# @return [Array<AdminDashboard::Reports::ResolvedReport>]- def self.list_all(search: nil, offset: 0, limit: nil)+ def self.list_all(search: nil, after: nil, limit: nil)raise NotImplementedErrorend+ # Sort + keyset-filter an in-memory set of resolved reports for+ # `list_all`. Suitable for providers small enough to materialise their+ # whole set (e.g. core reports); SQL-backed providers should push the+ # keyset into the query instead.+ def self.seek(reports, after:, limit:)+ reports = reports.sort_by { |report| sort_key(report) }+ if after+ threshold = [after[:title].to_s.downcase, after[:key].to_s]+ reports = reports.select { |report| (sort_key(report) <=> threshold) == 1 }+ end+ limit ? reports.first(limit) : reports+ end++ def self.sort_key(report)+ [report.title.to_s.downcase, report.key]+ end+# Identifiers from the input that the guardian is allowed to mount /# interact with. Default implementation is a subset of `resolve_many`# keys, which works for any provider whose access check is identicaldiff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rbindex e4874d06a93..d80bb9809b9 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/admin_dashboard_report_provider.rb@@ -21,36 +21,21 @@ module DiscourseDataExplorerendend- def self.list_all(search: nil, offset: 0, limit: nil)- persisted_total = persisted_scope(search: search).count- unpersisted = Query.unpersisted_defaults(search: search).sort_by { |q| q.name.to_s.downcase }-- results = []- remaining = limit-- if offset < persisted_total- take =- if remaining- [remaining, persisted_total - offset].min- else- persisted_total - offset- end- results.concat(persisted_scope(search: search).offset(offset).limit(take).to_a)- remaining -= results.size if remaining- end-- if remaining.nil? || remaining > 0- unpersisted_offset = [offset - persisted_total, 0].max- slice =- if remaining- unpersisted[unpersisted_offset, remaining]- else- unpersisted[unpersisted_offset..]- end- results.concat(Array(slice))- end-- results.map { |q| build_resolved(q) }+ def self.list_all(search: nil, after: nil, limit: nil)+ persisted =+ persisted_after(search: search, after: after, limit: limit).map do |query|+ build_resolved(query)+ end+ unpersisted =+ seek(+ Query.unpersisted_defaults(search: search).map { |query| build_resolved(query) },+ after: after,+ limit: limit,+ )++ merged =+ (persisted + unpersisted).sort_by { |report| [report.title.to_s.downcase, report.key] }+ limit ? merged.first(limit) : mergedenddef self.fetch_many(identifiers, guardian:, filters: {})@@ -104,13 +89,34 @@ module DiscourseDataExplorerendprivate_class_method :load_queries- def self.persisted_scope(search:)- scope = Query.where(hidden: false).includes(:groups).order(:name)- scope = scope.where(<<~SQL, s: "%#{Query.sanitize_sql_like(search)}%") if search.present?- name ILIKE :s OR description ILIKE :s- SQL- scope+ def self.persisted_after(search:, after:, limit:)+ scope = Query.where(hidden: false)+ if search.present?+ scope =+ scope.where(+ "name ILIKE :pattern OR description ILIKE :pattern",+ pattern: "%#{Query.sanitize_sql_like(search)}%",+ )+ end+ if after+ scope =+ scope.where(+ <<~SQL,+ LOWER(name) COLLATE "C" > LOWER(:after_title)+ OR (+ LOWER(name) = LOWER(:after_title)+ AND (:prefix || id::text) COLLATE "C" > :after_key+ )+ SQL+ after_title: after[:title],+ after_key: after[:key],+ prefix: "#{SOURCE_NAME}:",+ )+ end+ scope = scope.order(Arel.sql('LOWER(name) COLLATE "C" ASC, id::text COLLATE "C" ASC'))+ scope = scope.limit(limit) if limit+ scope.to_aend- private_class_method :persisted_scope+ private_class_method :persisted_afterendenddiff --git a/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rb b/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rbindex de10652d4cd..6f7afe022a6 100644--- a/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rb+++ b/plugins/discourse-data-explorer/spec/lib/discourse_data_explorer/admin_dashboard_report_provider_spec.rb@@ -108,13 +108,43 @@ RSpec.describe DiscourseDataExplorer::AdminDashboardReportProvider doexpect(described_class.list_all(search: "zz_no_match_zz")).to be_emptyend- it "respects offset and limit, paginating across persisted + defaults" do- first = described_class.list_all(offset: 0, limit: 1)- second = described_class.list_all(offset: 1, limit: 1)+ it "returns a title-sorted page and resumes after the cursor across persisted + defaults" do+ all = described_class.list_all+ titles = all.map { |report| report.title.to_s.downcase }+ expect(titles).to eq(titles.sort)+ first = described_class.list_all(limit: 1)expect(first.size).to eq(1)++ after = { title: first.first.title, key: first.first.key }+ second = described_class.list_all(after: after, limit: 1)+expect(second.size).to eq(1)- expect(first.first.identifier).not_to eq(second.first.identifier)+ expect(second.first.identifier).not_to eq(first.first.identifier)+ expect(+ described_class.sort_key(second.first) <=> described_class.sort_key(first.first),+ ).to eq(1)+ end++ it "orders by byte value so an emoji-titled query never re-appears after its own cursor" do+ lettered =+ Fabricate(:query, name: "alpha query", sql: "SELECT 1 AS value", hidden: false, user: admin)+ emoji =+ Fabricate(+ :query,+ name: "🦁 lion query",+ sql: "SELECT 1 AS value",+ hidden: false,+ user: admin,+ )++ identifiers = described_class.list_all.map(&:identifier)+ expect(identifiers.index(lettered.id.to_s)).to be < identifiers.index(emoji.id.to_s)++ after = { title: emoji.name, key: "data_explorer_query:#{emoji.id}" }+ expect(described_class.list_all(after: after).map(&:identifier)).not_to include(+ lettered.id.to_s,+ )endenddiff --git a/spec/lib/admin_dashboard/reports/core_report_provider_spec.rb b/spec/lib/admin_dashboard/reports/core_report_provider_spec.rbindex e2923306f19..0c4443ac3b7 100644--- a/spec/lib/admin_dashboard/reports/core_report_provider_spec.rb+++ b/spec/lib/admin_dashboard/reports/core_report_provider_spec.rb@@ -64,13 +64,21 @@ RSpec.describe AdminDashboard::Reports::CoreReportProvider doexpect(described_class.list_all(search: "zzzz_no_match_zzzz")).to be_emptyend- it "respects offset and limit" do- first_two = described_class.list_all(offset: 0, limit: 2)- next_two = described_class.list_all(offset: 2, limit: 2)-+ it "returns a title-sorted page and resumes after the cursor" do+ first_two = described_class.list_all(limit: 2)expect(first_two.size).to eq(2)++ titles = described_class.list_all.map { |report| report.title.to_s.downcase }+ expect(titles).to eq(titles.sort)++ after = { title: first_two.last.title, key: first_two.last.key }+ next_two = described_class.list_all(after: after, limit: 2)+expect(next_two.size).to eq(2)expect(first_two.map(&:identifier)).not_to include(*next_two.map(&:identifier))+ expect(+ described_class.sort_key(next_two.first) <=> described_class.sort_key(first_two.last),+ ).to eq(1)endenddiff --git a/spec/requests/admin/dashboard_controller_spec.rb b/spec/requests/admin/dashboard_controller_spec.rbindex 4e7958e7fdd..fadef8d64fb 100644--- a/spec/requests/admin/dashboard_controller_spec.rb+++ b/spec/requests/admin/dashboard_controller_spec.rb@@ -940,11 +940,11 @@ RSpec.describe Admin::DashboardController dodef self.label = "Fake"def self.universe- %w[a b c].map do |id|+ [%w[banana Banana], %w[date Date], %w[fig Fig]].map do |id, fruit|AdminDashboard::Reports::ResolvedReport.new(source: source_name,identifier: id,- title: "Fakey #{id}",+ title: "Zfruit #{fruit}",description: "Desc #{id}",label: label,url: "/fake/#{id}",@@ -952,13 +952,12 @@ RSpec.describe Admin::DashboardController doendend- def self.list_all(search: nil, offset: 0, limit: nil)+ def self.list_all(search: nil, after: nil, limit: nil)items = universeif search.present?items = items.select { |item| item.title.downcase.include?(search.downcase) }end- sliced = limit ? items[offset, limit] : items[offset..]- Array(sliced)+ seek(items, after: after, limit: limit)enddef self.resolve_many(identifiers, guardian:)@@ -969,28 +968,63 @@ RSpec.describe Admin::DashboardController doendend+ let(:alt_provider) do+ Class.new(AdminDashboard::Reports::SourceProvider) do+ def self.source_name = "b_alt_source"+ def self.label = "Alt"++ def self.universe+ [%w[apple Apple], %w[cherry Cherry], %w[egg Egg]].map do |id, fruit|+ AdminDashboard::Reports::ResolvedReport.new(+ source: source_name,+ identifier: id,+ title: "Zfruit #{fruit}",+ description: nil,+ label: label,+ url: nil,+ )+ end+ end++ def self.list_all(search: nil, after: nil, limit: nil)+ items = universe+ if search.present?+ items = items.select { |item| item.title.downcase.include?(search.downcase) }+ end+ seek(items, after: after, limit: limit)+ end++ def self.resolve_many(_identifiers, guardian:)+ {}+ end+ end+ end+let(:wide_provider) doClass.new(AdminDashboard::Reports::SourceProvider) do- def self.source_name = "a_wide_source"+ def self.source_name = "c_wide_source"def self.label = "Wide"- def self.list_all(search: nil, offset: 0, limit: nil)- items =- (1..40).map do |index|- AdminDashboard::Reports::ResolvedReport.new(- source: source_name,- identifier: "row_#{index}",- title: "Wideprefix Row #{index}",- description: nil,- label: label,- url: nil,- )- end+ def self.universe+ (1..40).map do |index|+ padded = format("%02d", index)+ AdminDashboard::Reports::ResolvedReport.new(+ source: source_name,+ identifier: "row_#{padded}",+ title: "Widerow #{padded}",+ description: nil,+ label: label,+ url: nil,+ )+ end+ end++ def self.list_all(search: nil, after: nil, limit: nil)+ items = universeif search.present?items = items.select { |item| item.title.downcase.include?(search.downcase) }end- sliced = limit ? items[offset, limit] : items[offset..]- Array(sliced)+ seek(items, after: after, limit: limit)enddef self.resolve_many(_identifiers, guardian:)@@ -1003,7 +1037,7 @@ RSpec.describe Admin::DashboardController doafter doDiscoursePluginRegistry._raw_admin_dashboard_report_sources.reject! do |entry|- [fake_provider, wide_provider].include?(entry[:value])+ [fake_provider, alt_provider, wide_provider].include?(entry[:value])endend@@ -1028,9 +1062,9 @@ RSpec.describe Admin::DashboardController doit "returns enabled, available, and providers" doDiscoursePluginRegistry.register_admin_dashboard_report_source(fake_provider, plugin)- AdminDashboardReport.create!(source: "a_fake_source", identifier: "b", position: 0)+ AdminDashboardReport.create!(source: "a_fake_source", identifier: "banana", position: 0)- get "/admin/dashboard/reports/available.json", params: { search: "Fakey" }+ get "/admin/dashboard/reports/available.json", params: { search: "Zfruit" }expect(response.status).to eq(200)body = response.parsed_body@@ -1038,36 +1072,59 @@ RSpec.describe Admin::DashboardController dofake_summary = body["providers"].find { |provider| provider["source"] == "a_fake_source" }expect(fake_summary["label"]).to eq("Fake")- expect(body["enabled"].map { |item| item["identifier"] }).to eq(["b"])+ expect(body["enabled"].map { |item| item["identifier"] }).to eq(["banana"])expect(body["enabled"].first["label"]).to eq("Fake")fake_available = body["available"].select { |item| item["source"] == "a_fake_source" }- expect(fake_available.map { |item| item["identifier"] }).to contain_exactly("a", "b", "c")+ expect(fake_available.map { |item| item["identifier"] }).to contain_exactly(+ "banana",+ "date",+ "fig",+ )end- it "includes already-enabled identifiers in the all list" do+ it "includes already-enabled identifiers in the available list" doDiscoursePluginRegistry.register_admin_dashboard_report_source(fake_provider, plugin)- AdminDashboardReport.create!(source: "a_fake_source", identifier: "a", position: 0)- AdminDashboardReport.create!(source: "a_fake_source", identifier: "b", position: 1)+ AdminDashboardReport.create!(source: "a_fake_source", identifier: "banana", position: 0)+ AdminDashboardReport.create!(source: "a_fake_source", identifier: "date", position: 1)- get "/admin/dashboard/reports/available.json", params: { search: "Fakey" }+ get "/admin/dashboard/reports/available.json", params: { search: "Zfruit" }body = response.parsed_bodyfake_available = body["available"].select { |item| item["source"] == "a_fake_source" }- expect(fake_available.map { |item| item["identifier"] }).to contain_exactly("a", "b", "c")+ expect(fake_available.map { |item| item["identifier"] }).to contain_exactly(+ "banana",+ "date",+ "fig",+ )+ end++ it "interleaves providers alphabetically by title rather than grouping them" do+ DiscoursePluginRegistry.register_admin_dashboard_report_source(fake_provider, plugin)+ DiscoursePluginRegistry.register_admin_dashboard_report_source(alt_provider, plugin)++ get "/admin/dashboard/reports/available.json", params: { search: "Zfruit" }+ body = response.parsed_body++ expect(body["available"].map { |item| item["identifier"] }).to eq(+ %w[apple banana cherry date egg fig],+ )+ expect(body["available"].map { |item| item["source"] }).to eq(+ %w[b_alt_source a_fake_source b_alt_source a_fake_source b_alt_source a_fake_source],+ )endit "paginates 30 items per response and emits a cursor when more exist" doDiscoursePluginRegistry.register_admin_dashboard_report_source(wide_provider, plugin)- get "/admin/dashboard/reports/available.json", params: { search: "Wideprefix" }+ get "/admin/dashboard/reports/available.json", params: { search: "Widerow" }expect(response.status).to eq(200)body = response.parsed_body- wide = body["available"].select { |item| item["source"] == "a_wide_source" }+ wide = body["available"].select { |item| item["source"] == "c_wide_source" }expect(wide.size).to eq(AdminDashboard::Reports::Listing::PAGE_SIZE)- expect(wide.first["identifier"]).to eq("row_1")+ expect(wide.first["identifier"]).to eq("row_01")expect(body["has_more"]).to eq(true)- expect(body["cursor"]).to eq("a_wide_source:30")+ expect(body["cursor"]).to eq("title" => "Widerow 30", "key" => "c_wide_source:row_30")endit "returns the next batch when the cursor from a previous response is sent back" do@@ -1075,48 +1132,43 @@ RSpec.describe Admin::DashboardController doget "/admin/dashboard/reports/available.json",params: {- search: "Wideprefix",- cursor: "a_wide_source:30",+ search: "Widerow",+ cursor: {+ title: "Widerow 30",+ key: "c_wide_source:row_30",+ },}body = response.parsed_body- wide = body["available"].select { |item| item["source"] == "a_wide_source" }+ wide = body["available"].select { |item| item["source"] == "c_wide_source" }- expect(wide.map { |item| item["identifier"] }).to eq((31..40).map { |n| "row_#{n}" })+ expect(wide.map { |item| item["identifier"] }).to eq(+ (31..40).map { |number| "row_#{number}" },+ )expect(body["has_more"]).to eq(false)expect(body["cursor"]).to be_nilend- it "ignores a cursor whose source is not registered" do+ it "treats a malformed cursor as the first page" doDiscoursePluginRegistry.register_admin_dashboard_report_source(wide_provider, plugin)- get "/admin/dashboard/reports/available.json", params: { cursor: "ghost_source:0" }-- expect(response.parsed_body["available"]).to be_empty- expect(response.parsed_body["has_more"]).to eq(false)- expect(response.parsed_body["cursor"]).to be_nil- end-- it "crosses provider boundaries via cursor instead of asking each provider at the same offset" do- DiscoursePluginRegistry.register_admin_dashboard_report_source(fake_provider, plugin)- DiscoursePluginRegistry.register_admin_dashboard_report_source(wide_provider, plugin)-- get "/admin/dashboard/reports/available.json", params: { cursor: "a_fake_source:0" }+ get "/admin/dashboard/reports/available.json",+ params: {+ search: "Widerow",+ cursor: "garbage",+ }body = response.parsed_body- sources = body["available"].map { |item| item["source"] }- expect(sources.first(3)).to eq(%w[a_fake_source a_fake_source a_fake_source])- expect(sources.last).to eq("a_wide_source")+ expect(body["available"].first["identifier"]).to eq("row_01")expect(body["has_more"]).to eq(true)- expect(body["cursor"]).to start_with("a_wide_source:")endit "filters by the search param" doDiscoursePluginRegistry.register_admin_dashboard_report_source(fake_provider, plugin)- get "/admin/dashboard/reports/available.json", params: { search: "Fakey a" }+ get "/admin/dashboard/reports/available.json", params: { search: "Zfruit ba" }body = response.parsed_bodyfake_available = body["available"].select { |item| item["source"] == "a_fake_source" }- expect(fake_available.map { |item| item["identifier"] }).to eq(["a"])+ expect(fake_available.map { |item| item["identifier"] }).to eq(["banana"])endendend