A platform for community discussion. Free, open, simple.
PERF: parallelize report fetching in the dashboard Reports section (#42252)
Osama Sayeghfda44c33e764
diff --git a/app/assets/stylesheets/admin/dashboard/reports.scss b/app/assets/stylesheets/admin/dashboard/reports.scssindex 2ec51ca2761..34d6c2016a8 100644--- a/app/assets/stylesheets/admin/dashboard/reports.scss+++ b/app/assets/stylesheets/admin/dashboard/reports.scss@@ -76,9 +76,17 @@width: 100% !important; // no other way to overrule the JS inline stylingheight: 100% !important;}++ .loading-container {+ display: flex;+ align-items: center;+ justify-content: center;+ height: 100%;+ }}- &__empty {+ &__empty,+ &__error {display: flex;flex-direction: column;align-items: center;@@ -92,6 +100,10 @@}}+ &__error {+ color: var(--danger);+ }+&__add-report {border: 1px dashed var(--content-border-color);border-radius: var(--d-border-radius-large);diff --git a/config/locales/client.en.yml b/config/locales/client.en.ymlindex 7e91f478ea7..693e87fbccc 100644--- a/config/locales/client.en.yml+++ b/config/locales/client.en.yml@@ -7420,6 +7420,7 @@ en:add: "Add report"remove: "Remove report"no_data: "No data to display"+ report_error: "Couldn't load this report"modal:title: "Manage reports"search_placeholder: "Search reports"diff --git a/frontend/discourse/admin/components/dashboard/report-error-state.gjs b/frontend/discourse/admin/components/dashboard/report-error-state.gjsnew file mode 100644index 00000000000..2edcf7edfff--- /dev/null+++ b/frontend/discourse/admin/components/dashboard/report-error-state.gjs@@ -0,0 +1,9 @@+import dIcon from "discourse/ui-kit/helpers/d-icon";+import { i18n } from "discourse-i18n";++export default <template>+ <div class="db-report__error" role="alert">+ {{dIcon "triangle-exclamation"}}+ <span>{{i18n "admin.dashboard.reports_section.report_error"}}</span>+ </div>+</template>diff --git a/frontend/discourse/admin/components/dashboard/reports.gjs b/frontend/discourse/admin/components/dashboard/reports.gjsindex 2dc378d11b1..5266a27f889 100644--- a/frontend/discourse/admin/components/dashboard/reports.gjs+++ b/frontend/discourse/admin/components/dashboard/reports.gjs@@ -6,6 +6,7 @@ import { action } from "@ember/object";import didUpdate from "@ember/render-modifiers/modifiers/did-update";import { service } from "@ember/service";import DashboardReportEmptyState from "discourse/admin/components/dashboard/report-empty-state";+import DashboardReportErrorState from "discourse/admin/components/dashboard/report-error-state";import DashboardSection from "discourse/admin/components/dashboard/section";import ManageReports from "discourse/admin/components/modal/manage-reports";import { lookupAdminDashboardReportRenderer } from "discourse/admin/lib/admin-dashboard-report-renderers";@@ -14,6 +15,7 @@ import PluginOutlet from "discourse/components/plugin-outlet";import lazyHash from "discourse/helpers/lazy-hash";import { ajax } from "discourse/lib/ajax";import { popupAjaxError } from "discourse/lib/ajax-error";+import DConditionalLoadingSpinner from "discourse/ui-kit/d-conditional-loading-spinner";import dConcatClass from "discourse/ui-kit/helpers/d-concat-class";import dIcon from "discourse/ui-kit/helpers/d-icon";import { i18n } from "discourse-i18n";@@ -29,7 +31,12 @@ export default class DashboardReports extends Component {constructor() {super(...arguments);- this.cards = this.items.map((item) => ({ ...item, payload: null }));+ this.cards = this.items.map((item) => ({+ ...item,+ payload: null,+ error: false,+ isLoading: true,+ }));this.loadPayloads();}@@ -73,15 +80,26 @@ export default class DashboardReports extends Component {this.loading = true;try {- const payloads = await loadDashboardReports({+ const results = await loadDashboardReports({items: this.layoutItems,filters: this.filters,});+ this.cards = this.items.map((item) => {+ const result = results.get(item.key);+ return {+ ...item,+ payload: result?.payload ?? null,+ error: result?.error ?? false,+ isLoading: false,+ };+ });+ } catch (e) {this.cards = this.items.map((item) => ({...item,- payload: payloads.get(item.key),+ payload: null,+ error: true,+ isLoading: false,}));- } catch (e) {popupAjaxError(e);} finally {this.loading = false;@@ -157,10 +175,15 @@ export default class DashboardReports extends Component {{{/if}}</div><div class="db-report__chart">- {{#if card.payload}}- {{#if card.payload.empty}}+ <DConditionalLoadingSpinner+ @condition={{card.isLoading}}+ @size="small"+ >+ {{#if card.error}}+ <DashboardReportErrorState />+ {{else if card.payload.empty}}<DashboardReportEmptyState />- {{else}}+ {{else if card.payload}}{{#let(lookupAdminDashboardReportRenderer card.source)as |Renderer|@@ -177,7 +200,7 @@ export default class DashboardReports extends Component {{{/if}}{{/let}}{{/if}}- {{/if}}+ </DConditionalLoadingSpinner></div></div>{{/each}}diff --git a/frontend/discourse/admin/lib/dashboard-reports-loader.js b/frontend/discourse/admin/lib/dashboard-reports-loader.jsindex 77e7973a609..b32aad46f2a 100644--- a/frontend/discourse/admin/lib/dashboard-reports-loader.js+++ b/frontend/discourse/admin/lib/dashboard-reports-loader.js@@ -13,7 +13,7 @@ export async function loadDashboardReports({ items, filters }) {const map = new Map();for (const entry of response.items) {- map.set(entry.key, entry.data);+ map.set(entry.key, { payload: entry.data, error: Boolean(entry.error) });}return map;}diff --git a/frontend/discourse/tests/integration/components/dashboard/reports-test.gjs b/frontend/discourse/tests/integration/components/dashboard/reports-test.gjsindex 862d70bfe22..b6175d0eb34 100644--- a/frontend/discourse/tests/integration/components/dashboard/reports-test.gjs+++ b/frontend/discourse/tests/integration/components/dashboard/reports-test.gjs@@ -1,9 +1,53 @@-import { render } from "@ember/test-helpers";+import { hash } from "@ember/helper";+import { render, settled, waitFor } from "@ember/test-helpers";import { module, test } from "qunit";import DashboardReports from "discourse/admin/components/dashboard/reports";+import { registerAdminDashboardReportRenderer } from "discourse/admin/lib/admin-dashboard-report-renderers";import { setupRenderingTest } from "discourse/tests/helpers/component-test";+import pretender, { response } from "discourse/tests/helpers/create-pretender";import { i18n } from "discourse-i18n";+const FakeReportRenderer = <template>+ <div class="fake-report-renderer">{{@payload.type}}</div>+</template>;++const ITEMS = [+ {+ source: "test_source",+ identifier: "signups",+ key: "test_source:signups",+ title: "Signups",+ description: "New account signups",+ label: null,+ url: "/admin/reports/signups",+ },+ {+ source: "core_report",+ identifier: "broken",+ key: "core_report:broken",+ title: "Broken report",+ description: "Always fails to load",+ label: null,+ url: "/admin/reports/broken",+ },+ {+ source: "core_report",+ identifier: "empty_report",+ key: "core_report:empty_report",+ title: "Empty report",+ description: "Never has data",+ label: null,+ url: "/admin/reports/empty_report",+ },+];++function deferredBulkFetch() {+ let resolve;+ const promise = new Promise((r) => (resolve = r));+ pretender.post("/admin/dashboard/reports/bulk", () => promise);+ return { resolve: (body) => resolve(response(body)) };+}+module("Integration | Component | DashboardReports", function (hooks) {setupRenderingTest(hooks);@@ -25,4 +69,96 @@ module("Integration | Component | DashboardReports", function (hooks) {.dom(".db-report__add-report").doesNotExist("the add report button is not rendered");});++ test("shows a per-card loading spinner, then each card's own success, empty, or error state once the bulk fetch resolves", async function (assert) {+ registerAdminDashboardReportRenderer("test_source", FakeReportRenderer);++ const { resolve } = deferredBulkFetch();++ const renderPromise = render(+ <template><DashboardReports @data={{hash items=ITEMS}} /></template>+ );++ await waitFor(".db-report__card .loading-container.visible");+ assert+ .dom(".db-report__card .loading-container.visible")+ .exists(+ { count: 3 },+ "every card shows a loading spinner before the fetch resolves"+ );++ resolve({+ items: [+ {+ source: "test_source",+ identifier: "signups",+ key: "test_source:signups",+ data: { type: "signups", empty: false },+ },+ {+ source: "core_report",+ identifier: "broken",+ key: "core_report:broken",+ data: null,+ error: true,+ },+ {+ source: "core_report",+ identifier: "empty_report",+ key: "core_report:empty_report",+ data: { type: "empty_report", data: [], empty: true },+ },+ ],+ });+ await renderPromise;+ await settled();++ assert+ .dom(".db-report__card .loading-container.visible")+ .doesNotExist(+ "no card is left in a loading state once the fetch resolves"+ );++ assert+ .dom('[data-identifier="core_report:broken"] .db-report__error')+ .hasText(+ i18n("admin.dashboard.reports_section.report_error"),+ "the failed report shows its own error state"+ );+ assert+ .dom('[data-identifier="test_source:signups"] .db-report__error')+ .doesNotExist(+ "a report that loaded successfully does not show an error state"+ );++ assert+ .dom('[data-identifier="core_report:empty_report"] .db-report__empty')+ .exists("a report with no data shows the empty state");+ assert+ .dom('[data-identifier="test_source:signups"] .db-report__empty')+ .doesNotExist("a report with data does not show the empty state");+ assert+ .dom('[data-identifier="test_source:signups"] .fake-report-renderer')+ .hasText("signups", "a successful report renders its content");+ });++ test("clears every card's loading state when the whole bulk request fails", async function (assert) {+ pretender.post("/admin/dashboard/reports/bulk", () => response(500, {}));++ await render(+ <template><DashboardReports @data={{hash items=ITEMS}} /></template>+ );++ assert+ .dom(".db-report__card .loading-container.visible")+ .doesNotExist(+ "no card is left spinning once the whole request has failed"+ );+ assert+ .dom(".db-report__card .db-report__error")+ .exists(+ { count: ITEMS.length },+ "every card shows its own error state instead"+ );+ });});diff --git a/frontend/discourse/tests/unit/lib/dashboard-reports-loader-test.js b/frontend/discourse/tests/unit/lib/dashboard-reports-loader-test.jsnew file mode 100644index 00000000000..f97fbdefec9--- /dev/null+++ b/frontend/discourse/tests/unit/lib/dashboard-reports-loader-test.js@@ -0,0 +1,59 @@+import { setupTest } from "ember-qunit";+import { module, test } from "qunit";+import { loadDashboardReports } from "discourse/admin/lib/dashboard-reports-loader";+import pretender, { response } from "discourse/tests/helpers/create-pretender";++module("Unit | Lib | dashboard-reports-loader", function (hooks) {+ setupTest(hooks);++ test("returns an empty map without making a request when there are no items", async function (assert) {+ pretender.post("/admin/dashboard/reports/bulk", () => {+ throw new Error("should not be called");+ });++ const result = await loadDashboardReports({ items: [], filters: {} });++ assert.strictEqual(result.size, 0);+ });++ test("maps each response entry to its payload and error state, keyed by item key", async function (assert) {+ pretender.post("/admin/dashboard/reports/bulk", () =>+ response({+ items: [+ {+ source: "core_report",+ identifier: "signups",+ key: "core_report:signups",+ data: { type: "signups" },+ },+ {+ source: "core_report",+ identifier: "broken",+ key: "core_report:broken",+ data: null,+ error: true,+ },+ ],+ })+ );++ const result = await loadDashboardReports({+ items: [+ { source: "core_report", identifier: "signups" },+ { source: "core_report", identifier: "broken" },+ ],+ filters: {},+ });++ assert.deepEqual(+ result.get("core_report:signups"),+ { payload: { type: "signups" }, error: false },+ "a successful item has its payload and error: false"+ );+ assert.deepEqual(+ result.get("core_report:broken"),+ { payload: null, error: true },+ "a failed item has a null payload and error: true"+ );+ });+});diff --git a/lib/admin_dashboard/reports/bulk_fetch.rb b/lib/admin_dashboard/reports/bulk_fetch.rbindex 0f5eb2d7f82..c4d366f637f 100644--- a/lib/admin_dashboard/reports/bulk_fetch.rb+++ b/lib/admin_dashboard/reports/bulk_fetch.rb@@ -3,25 +3,85 @@module AdminDashboardmodule Reportsclass BulkFetch+ FETCH_FAILED = :fetch_failed++ MAX_CONCURRENT_FETCHES = 4+def self.call(items:, filters:, guardian:)- per_source =- AdminDashboard::Reports::Registry.dispatch_per_source(items) do |provider, group|- identifiers = group.map { |i| i[:identifier] }- provider.fetch_many(identifiers, guardian:, filters:)- end+ payloads = fetch_in_parallel(items, guardian:, filters:)results =- items.map do |item|- {- source: item[:source],- identifier: item[:identifier],- key: "#{item[:source]}:#{item[:identifier]}",- data: per_source.dig(item[:source], item[:identifier]),- }- end+ items+ .zip(payloads)+ .map do |item, payload|+ failed = payload == FETCH_FAILED+ errored = failed || (payload.is_a?(Hash) && payload[:error].present?)+ {+ source: item[:source],+ identifier: item[:identifier],+ key: item_key(item),+ data: failed ? nil : payload,+ error: errored,+ }+ end{ items: results }end++ def self.pool_size+ available = [ActiveRecord::Base.connection_pool.size - 1, 1].max+ [MAX_CONCURRENT_FETCHES, available].min+ end++ def self.thread_pool+ @thread_pool ||=+ Scheduler::ThreadPool.new(min_threads: 0, max_threads: pool_size, idle_time: 30)+ end++ def self.fetch_in_parallel(items, guardian:, filters:)+ return [] if items.empty?++ queue = Queue.new++ items.each_with_index do |item, index|+ thread_pool.post do+ ActiveRecord::Base.with_connection(prevent_permanent_checkout: true) do+ queue << [index, fetch_item(item, guardian:, filters:)]+ end+ rescue StandardError => e+ Discourse.warn_exception(+ e,+ message: "Failed to fetch admin dashboard report",+ env: {+ source: item[:source],+ identifier: item[:identifier],+ },+ )+ queue << [index, FETCH_FAILED]+ end+ end++ results = Array.new(items.size)+ items.size.times do+ index, result = queue.pop+ results[index] = result+ end+ results+ end+ private_class_method :fetch_in_parallel++ def self.item_key(item)+ "#{item[:source]}:#{item[:identifier]}"+ end+ private_class_method :item_key++ def self.fetch_item(item, guardian:, filters:)+ provider = AdminDashboard::Reports::Registry.provider_for(item[:source])+ return nil if provider.nil?++ provider.fetch_many([item[:identifier]], guardian:, filters:)[item[:identifier]]+ end+ private_class_method :fetch_itemendendenddiff --git a/lib/admin_dashboard/reports/source_provider.rb b/lib/admin_dashboard/reports/source_provider.rbindex 3dc0d73b680..4a4526cffce 100644--- a/lib/admin_dashboard/reports/source_provider.rb+++ b/lib/admin_dashboard/reports/source_provider.rb@@ -41,6 +41,11 @@ module AdminDashboard# Expensive data fetch. Called by the bulk endpoint to load chart/table# content. Providers own their own caching.#+ # A payload hash may include a truthy `:error` key to report a failure+ # that happened while producing that identifier's data (e.g. a bad+ # query) without raising — the bulk endpoint surfaces this the same+ # way it surfaces an unhandled exception from `fetch_many` itself.+ ## @param identifiers [Array<String>]# @param guardian [Guardian]# @param filters [Hash] dashboard-level filter values (date range, etc).diff --git a/lib/scheduler/thread_pool.rb b/lib/scheduler/thread_pool.rbindex 7ced614f65d..b4a39ed8b1c 100644--- a/lib/scheduler/thread_pool.rb+++ b/lib/scheduler/thread_pool.rb@@ -48,7 +48,8 @@ module Schedulerraise ShutdownError, "Cannot post work to a shutdown ThreadPool" if shutdown?db = RailsMultisite::ConnectionManagement.current_db- wrapped_block = wrap_block(block, db)+ locale = I18n.locale+ wrapped_block = wrap_block(block, db, locale)@mutex.synchronize do@queue << wrapped_block@@ -112,9 +113,11 @@ module Schedulerprivate- def wrap_block(block, db)+ def wrap_block(block, db, locale)proc do- RailsMultisite::ConnectionManagement.with_connection(db) { block.call }+ RailsMultisite::ConnectionManagement.with_connection(db) do+ I18n.with_locale(locale) { block.call }+ endrescue StandardError => eDiscourse.warn_exception(e, message: "Discourse Scheduler ThreadPool: Unhandled exception")enddiff --git a/spec/lib/admin_dashboard/reports/bulk_fetch_spec.rb b/spec/lib/admin_dashboard/reports/bulk_fetch_spec.rbnew file mode 100644index 00000000000..40aaff43b63--- /dev/null+++ b/spec/lib/admin_dashboard/reports/bulk_fetch_spec.rb@@ -0,0 +1,99 @@+# frozen_string_literal: true++RSpec.describe AdminDashboard::Reports::BulkFetch do+ fab!(:admin)+ let(:guardian) { admin.guardian }+ let(:plugin) { Plugin::Instance.new }+ let(:registered_providers) { [] }++ def register_provider(klass)+ registered_providers << klass+ DiscoursePluginRegistry.register_admin_dashboard_report_source(klass, plugin)+ end++ after do+ DiscoursePluginRegistry._raw_admin_dashboard_report_sources.reject! do |entry|+ registered_providers.include?(entry[:value])+ end+ if thread_pool = described_class.instance_variable_get(:@thread_pool)+ thread_pool.shutdown+ thread_pool.wait_for_termination(timeout: 1)+ described_class.remove_instance_variable(:@thread_pool)+ end+ end++ describe ".call" do+ it "fetches data from a real registered provider end-to-end" do+ result =+ described_class.call(+ items: [{ source: "core_report", identifier: "signups" }],+ filters: {+ },+ guardian: guardian,+ )++ expect(result[:items].size).to eq(1)+ item = result[:items].first+ expect(item).to include(source: "core_report", identifier: "signups", error: false)+ expect(item[:data][:type]).to eq("signups")+ end++ it "calls each item's provider individually so multiple items of the same source still both resolve" do+ single_item_only_provider =+ Class.new(AdminDashboard::Reports::SourceProvider) do+ define_singleton_method(:source_name) { "single_item_only_source" }+ define_singleton_method(:fetch_many) do |identifiers, guardian:, filters: {}|+ if identifiers.size != 1+ raise "expected exactly one identifier, got #{identifiers.size}"+ end+ { identifiers.first => { value: identifiers.first } }+ end+ end+ register_provider(single_item_only_provider)++ result =+ described_class.call(+ items: [+ { source: "single_item_only_source", identifier: "a" },+ { source: "single_item_only_source", identifier: "b" },+ ],+ filters: {+ },+ guardian: guardian,+ )++ by_identifier = result[:items].index_by { |item| item[:identifier] }+ expect(by_identifier["a"]).to include(data: { value: "a" }, error: false)+ expect(by_identifier["b"]).to include(data: { value: "b" }, error: false)+ end++ it "returns no items for an empty items array" do+ expect(described_class.call(items: [], filters: {}, guardian: guardian)).to eq(items: [])+ end++ it "treats a payload carrying its own error key as failed, without discarding the payload" do+ soft_failing_provider =+ Class.new(AdminDashboard::Reports::SourceProvider) do+ define_singleton_method(:source_name) { "soft_failing_source" }+ define_singleton_method(:fetch_many) do |identifiers, guardian:, filters: {}|+ identifiers.each_with_object({}) do |identifier, hash|+ hash[identifier] = { error: "syntax error at or near \"selct\"", empty: true }+ end+ end+ end+ register_provider(soft_failing_provider)++ result =+ described_class.call(+ items: [{ source: "soft_failing_source", identifier: "broken_query" }],+ filters: {+ },+ guardian: guardian,+ )++ item = result[:items].first+ expect(item[:error]).to eq(true)+ expect(item[:data]).to eq(error: "syntax error at or near \"selct\"", empty: true)+ end+ end+enddiff --git a/spec/lib/scheduler/thread_pool_spec.rb b/spec/lib/scheduler/thread_pool_spec.rbindex c62d01724d1..38965985408 100644--- a/spec/lib/scheduler/thread_pool_spec.rb+++ b/spec/lib/scheduler/thread_pool_spec.rb@@ -61,6 +61,14 @@ RSpec.describe Scheduler::ThreadPool, type: :multisite doexpect(completion_queue.pop).to eq("second")end+ it "maintains the posting thread's locale" do+ completion_queue = Queue.new++ I18n.with_locale(:fr) { pool.post { completion_queue << I18n.locale } }++ expect(completion_queue.pop).to eq(:fr)+ end+it "scales up threads when work increases" docompletion_queue = Queue.newblocker_queue = Queue.newdiff --git a/spec/requests/admin/dashboard_controller_spec.rb b/spec/requests/admin/dashboard_controller_spec.rbindex 16c3dab36fc..813afb817b6 100644--- a/spec/requests/admin/dashboard_controller_spec.rb+++ b/spec/requests/admin/dashboard_controller_spec.rb@@ -1219,11 +1219,23 @@ RSpec.describe Admin::DashboardController doendend+ let(:raising_provider) do+ Class.new(AdminDashboard::Reports::SourceProvider) do+ def self.source_name = "raising_source"+ def self.fetch_many(identifiers, guardian:, filters: {})+ identifiers.each_with_object({}) do |id, h|+ raise "boom" if id == "broken"+ h[id.to_s] = { id: id.to_s }+ end+ end+ end+ end+let(:plugin) { Plugin::Instance.new }after doDiscoursePluginRegistry._raw_admin_dashboard_report_sources.reject! do |entry|- entry[:value] == fake_provider+ entry[:value] == fake_provider || entry[:value] == raising_providerendend@@ -1280,7 +1292,7 @@ RSpec.describe Admin::DashboardController doexpect(items.map { |i| [i["identifier"], i["data"]["id"]] }).to eq([%w[a a], %w[b b]])end- it "returns data: nil for items whose source has no registered provider" do+ it "returns data: nil, error: false for items whose source has no registered provider" dopost "/admin/dashboard/reports/bulk.json",params: {items: [{ source: "totally_unregistered", identifier: "x" }],@@ -1290,9 +1302,27 @@ RSpec.describe Admin::DashboardController doitems = response.parsed_body["items"]expect(items.size).to eq(1)expect(items.first["data"]).to be_nil+ expect(items.first["error"]).to eq(false)expect(items.first["source"]).to eq("totally_unregistered")end+ it "returns error: true for an item whose provider raises, without affecting other items" do+ DiscoursePluginRegistry.register_admin_dashboard_report_source(raising_provider, plugin)++ post "/admin/dashboard/reports/bulk.json",+ params: {+ items: [+ { source: "raising_source", identifier: "broken" },+ { source: "raising_source", identifier: "ok" },+ ],+ }++ expect(response.status).to eq(200)+ items = response.parsed_body["items"].index_by { |item| item["identifier"] }+ expect(items["broken"]).to include("data" => nil, "error" => true)+ expect(items["ok"]).to include("data" => { "id" => "ok" }, "error" => false)+ end+it "returns data shaped by the dashboard filters" doDiscoursePluginRegistry.register_admin_dashboard_report_source(fake_provider, plugin)