A platform for community discussion. Free, open, simple.
PERF: Add fast nested reply stats preparation (#41870)
Mark VanLandinghamc85593eda88d
diff --git a/app/jobs/regular/prepare_nested_reply_stats.rb b/app/jobs/regular/prepare_nested_reply_stats.rbnew file mode 100644index 00000000000..bf603d58451--- /dev/null+++ b/app/jobs/regular/prepare_nested_reply_stats.rb@@ -0,0 +1,83 @@+# frozen_string_literal: true++module Jobs+ class PrepareNestedReplyStats < ::Jobs::Base+ sidekiq_options queue: "low"+ cluster_concurrency 1++ def execute(args = {})+ raise Discourse::ReadOnly if Discourse.readonly_mode?++ args ||= {}+ isolated_topic_id = args[:topic_id].to_i+ if isolated_topic_id.positive?+ Jobs::BackfillNestedReplyStats.backfill_topic(isolated_topic_id)+ return+ end++ after_topic_id = args[:after_topic_id].to_i+ max_topic_id = args[:max_topic_id].to_i+ max_topic_id = maximum_topic_id if max_topic_id.zero?++ if after_topic_id >= max_topic_id+ log_completion(max_topic_id)+ return+ end++ topic_ids = topic_ids_to_prepare(after_topic_id:, max_topic_id:)+ if topic_ids.empty?+ log_completion(max_topic_id)+ return+ end++ topic_ids.each do |topic_id|+ Jobs::BackfillNestedReplyStats.backfill_topic(topic_id)+ rescue => error+ Discourse.warn_exception(+ error,+ message: "Failed to prepare nested reply stats for topic #{topic_id}",+ )+ Jobs.enqueue(:prepare_nested_reply_stats, topic_id: topic_id)+ end+ after_topic_id = topic_ids.last++ if after_topic_id < max_topic_id+ Jobs.enqueue(:prepare_nested_reply_stats, after_topic_id:, max_topic_id:)+ else+ log_completion(max_topic_id)+ end+ end++ private++ def maximum_topic_id+ Topic.where(archetype: Archetype.default, deleted_at: nil).maximum(:id).to_i+ end++ def topic_ids_to_prepare(after_topic_id:, max_topic_id:)+ DB.query_single(+ <<~SQL,+ SELECT topics.id+ FROM topics+ INNER JOIN posts op+ ON op.topic_id = topics.id+ AND op.post_number = 1+ WHERE topics.id > :after_topic_id+ AND topics.id <= :max_topic_id+ AND topics.deleted_at IS NULL+ AND topics.archetype = :archetype+ ORDER BY topics.id+ LIMIT :batch_size+ SQL+ after_topic_id: after_topic_id,+ max_topic_id: max_topic_id,+ archetype: Archetype.default,+ batch_size: SiteSetting.nested_replies_backfill_batch_size,+ )+ end++ def log_completion(max_topic_id)+ Rails.logger.info("Nested reply stats preparation completed through topic #{max_topic_id}")+ end+ end+enddiff --git a/app/jobs/scheduled/backfill_nested_reply_stats.rb b/app/jobs/scheduled/backfill_nested_reply_stats.rbindex a74b3b0d6c8..abddc13b0d4 100644--- a/app/jobs/scheduled/backfill_nested_reply_stats.rb+++ b/app/jobs/scheduled/backfill_nested_reply_stats.rb@@ -13,7 +13,7 @@ module Jobstopic_ids = topic_ids_missing_stats(category_id: args[:category_id])return if topic_ids.empty?- topic_ids.each { |topic_id| backfill_topic(topic_id) }+ topic_ids.each { |topic_id| self.class.backfill_topic(topic_id) }endprivate@@ -59,7 +59,7 @@ module Jobs)end- def backfill_topic(topic_id)+ def self.backfill_topic(topic_id)DB.exec(<<~SQL, topic_id: topic_id, whisper_type: Post.types[:whisper])WITH RECURSIVEedges AS (diff --git a/app/models/concerns/has_nested_reply_stats.rb b/app/models/concerns/has_nested_reply_stats.rbindex 0cf50123513..63ea8e7fe39 100644--- a/app/models/concerns/has_nested_reply_stats.rb+++ b/app/models/concerns/has_nested_reply_stats.rb@@ -13,7 +13,7 @@ module HasNestedReplyStats# are left alone — their total descendant count is unchanged — while# ancestors unique to either chain are adjusted by this post's subtree size.def nested_replies_apply_reparent(previous_reply_to_post_number)- return unless SiteSetting.nested_replies_enabled+ return unless maintain_nested_reply_stats?return if previous_reply_to_post_number == reply_to_post_numbersubtree_size, whisper_subtree_size = nested_replies_subtree_sizes@@ -76,8 +76,12 @@ module HasNestedReplyStatsprivate+ def maintain_nested_reply_stats?+ SiteSetting.nested_replies_enabled || SiteSetting.nested_replies_stats_maintenance_enabled+ end+def nested_replies_increment_stats- return unless SiteSetting.nested_replies_enabled+ return unless maintain_nested_reply_stats?return if reply_to_post_number.blank?ancestors = nested_replies_walk(reply_to_post_number)@@ -110,7 +114,7 @@ module HasNestedReplyStatsenddef nested_replies_decrement_stats- return unless SiteSetting.nested_replies_enabled+ return unless maintain_nested_reply_stats?if reply_to_post_number.present?subtree_size, whisper_subtree_size = nested_replies_subtree_sizesis_whisper = post_type == Post.types[:whisper] ? 1 : 0diff --git a/config/locales/server.en.yml b/config/locales/server.en.ymlindex d99ef1b2677..7e5b491d6e1 100644--- a/config/locales/server.en.yml+++ b/config/locales/server.en.yml@@ -2951,7 +2951,8 @@ en:nested_replies_hot_preload_depth_penalty: "Priority subtracted for each level of depth when choosing nested replies to pre-expand in hot view."nested_replies_default_sort: "Default sort algorithm for nested replies view."nested_replies_max_depth: "Maximum visual nesting depth for threaded replies. Posts deeper than this show a 'Continue this thread' link."- nested_replies_backfill_batch_size: "Number of topics to backfill nested reply stats for per scheduled run."+ nested_replies_backfill_batch_size: "Number of topics to process per nested reply stats backfill batch."+ nested_replies_stats_maintenance_enabled: "Maintain nested reply statistics even while nested replies are disabled."nested_replies_cap_nesting_depth: "Cap reply nesting at the maximum visual depth. When enabled, replies to the deepest level are re-parented as siblings instead of children."enable_new_defaults_for_trust_level_3_requirements: "We have updated the TL3 default requirements so they’re more reasonable for members of large, active communities. If you have previously overridden these settings, this upcoming change will have no effect."floating_dismiss_topics_on_mobile: "Shows a floating 'Dismiss...' button on mobile for the topic list for easier access and to free up space in the top of the topic list"diff --git a/config/site_settings.yml b/config/site_settings.ymlindex 5c0cff0f191..62eb3b38450 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -2119,6 +2119,9 @@ nested_replies:min: 1max: 10000hidden: true+ nested_replies_stats_maintenance_enabled:+ default: false+ hidden: trueemail:email_time_window_mins:diff --git a/lib/tasks/nested_replies.rake b/lib/tasks/nested_replies.rakenew file mode 100644index 00000000000..fb412b7b0ed--- /dev/null+++ b/lib/tasks/nested_replies.rake@@ -0,0 +1,23 @@+# frozen_string_literal: true++namespace :nested_replies do+ desc "Prepare nested reply statistics for all regular topics"+ task prepare_stats: :environment do+ enqueue =+ lambda do+ database = RailsMultisite::ConnectionManagement.current_db+ SiteSetting.nested_replies_stats_maintenance_enabled = true+ max_topic_id = Topic.where(archetype: Archetype.default, deleted_at: nil).maximum(:id).to_i+ Jobs.enqueue(:prepare_nested_reply_stats, max_topic_id: max_topic_id)++ puts "Enqueued nested reply stats preparation for '#{database}' " \+ "through topic #{max_topic_id}"+ end++ if ENV["RAILS_DB"]+ enqueue.call+ else+ RailsMultisite::ConnectionManagement.each_connection { enqueue.call }+ end+ end+enddiff --git a/spec/jobs/prepare_nested_reply_stats_spec.rb b/spec/jobs/prepare_nested_reply_stats_spec.rbnew file mode 100644index 00000000000..782b5826956--- /dev/null+++ b/spec/jobs/prepare_nested_reply_stats_spec.rb@@ -0,0 +1,79 @@+# frozen_string_literal: true++RSpec.describe Jobs::PrepareNestedReplyStats do+ before do+ SiteSetting.nested_replies_enabled = false+ SiteSetting.nested_replies_default = false+ SiteSetting.nested_replies_stats_maintenance_enabled = false+ end++ it "prepares every regular topic while nested replies remain disabled" do+ topic = Fabricate(:topic)+ op = Fabricate(:post, topic: topic, post_number: 1)+ Fabricate(:post, topic: topic, reply_to_post_number: op.post_number)++ deleted_topic = Fabricate(:topic, deleted_at: Time.current)+ deleted_op = Fabricate(:post, topic: deleted_topic, post_number: 1)++ private_message = Fabricate(:private_message_topic)+ private_message_op = Fabricate(:post, topic: private_message, post_number: 1)++ described_class.new.execute(max_topic_id: Topic.maximum(:id))++ expect(NestedViewPostStat.find_by(post_id: op.id)).to have_attributes(+ direct_reply_count: 1,+ total_descendant_count: 1,+ )+ expect(NestedViewPostStat.exists?(post_id: deleted_op.id)).to eq(false)+ expect(NestedViewPostStat.exists?(post_id: private_message_op.id)).to eq(false)+ end++ it "immediately continues from the last topic in each bounded batch" do+ after_topic_id = Topic.maximum(:id).to_i+ first_topic = Fabricate(:topic)+ first_op = Fabricate(:post, topic: first_topic, post_number: 1)+ second_topic = Fabricate(:topic)+ second_op = Fabricate(:post, topic: second_topic, post_number: 1)+ max_topic_id = second_topic.id+ later_topic = Fabricate(:topic)+ later_op = Fabricate(:post, topic: later_topic, post_number: 1)+ SiteSetting.nested_replies_backfill_batch_size = 1++ described_class.new.execute(after_topic_id:, max_topic_id:)++ expect(NestedViewPostStat.exists?(post_id: first_op.id)).to eq(true)+ expect(NestedViewPostStat.exists?(post_id: second_op.id)).to eq(false)+ expect(described_class.jobs.size).to eq(1)+ expect(described_class.jobs.first["at"]).to be_nil+ continuation = described_class.jobs.first["args"].first.with_indifferent_access+ expect(continuation).to include(after_topic_id: first_topic.id, max_topic_id: max_topic_id)++ described_class.jobs.clear+ described_class.new.execute(continuation)++ expect(NestedViewPostStat.exists?(post_id: second_op.id)).to eq(true)+ expect(NestedViewPostStat.exists?(post_id: later_op.id)).to eq(false)+ expect(described_class.jobs).to be_empty+ end++ it "isolates a failed topic so later topics are still prepared" do+ after_topic_id = Topic.maximum(:id).to_i+ failed_topic = Fabricate(:topic)+ Fabricate(:post, topic: failed_topic, post_number: 1)+ healthy_topic = Fabricate(:topic)+ healthy_op = Fabricate(:post, topic: healthy_topic, post_number: 1)+ allow(Discourse).to receive(:warn_exception)+ allow(Jobs::BackfillNestedReplyStats).to receive(:backfill_topic).and_call_original+ allow(Jobs::BackfillNestedReplyStats).to receive(:backfill_topic).with(+ failed_topic.id,+ ).and_raise("boom")++ described_class.new.execute(after_topic_id:, max_topic_id: healthy_topic.id)++ expect(NestedViewPostStat.exists?(post_id: healthy_op.id)).to eq(true)+ expect(described_class.jobs.size).to eq(1)+ retry_args = described_class.jobs.first["args"].first.with_indifferent_access+ expect(retry_args).to include(topic_id: failed_topic.id)+ expect { described_class.new.execute(retry_args) }.to raise_error("boom")+ end+enddiff --git a/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rb b/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rbindex 4861111bddd..27b414701cd 100644--- a/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rb+++ b/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rb@@ -11,9 +11,10 @@ RSpec.describe Jobs::BackfillNestedReplyStats dodescribed_class.new.execute(args)end- it "does nothing when feature is disabled" do+ it "does nothing when feature is disabled, even during explicit preparation" doSiteSetting.nested_replies_enabled = falseFabricate(:post, topic: topic, reply_to_post_number: 1)+ SiteSetting.nested_replies_stats_maintenance_enabled = trueexecutediff --git a/spec/lib/nested_replies/nested_replies_n_plus_one_spec.rb b/spec/lib/nested_replies/nested_replies_n_plus_one_spec.rbindex 0eaa57bfdd0..23c91f39c1a 100644--- a/spec/lib/nested_replies/nested_replies_n_plus_one_spec.rb+++ b/spec/lib/nested_replies/nested_replies_n_plus_one_spec.rb@@ -32,6 +32,23 @@ RSpec.describe "Nested replies N+1 elimination", type: :request doenddescribe "after_create stats increment" do+ it "maintains stats during preparation while nested replies are disabled" do+ SiteSetting.nested_replies_enabled = false+ SiteSetting.nested_replies_stats_maintenance_enabled = true++ parent = Fabricate(:post, topic: topic, user: user, reply_to_post_number: op.post_number)+ Fabricate(:post, topic: topic, user: user, reply_to_post_number: parent.post_number)++ expect(NestedViewPostStat.find_by(post_id: op.id)).to have_attributes(+ direct_reply_count: 1,+ total_descendant_count: 2,+ )+ expect(NestedViewPostStat.find_by(post_id: parent.id)).to have_attributes(+ direct_reply_count: 1,+ total_descendant_count: 1,+ )+ end+it "uses constant queries regardless of chain depth" dochain_3 = create_reply_chain(depth: 3)queries_3 =diff --git a/spec/tasks/nested_replies_spec.rb b/spec/tasks/nested_replies_spec.rbnew file mode 100644index 00000000000..08525797e5c--- /dev/null+++ b/spec/tasks/nested_replies_spec.rb@@ -0,0 +1,17 @@+# frozen_string_literal: true++RSpec.describe "nested_replies tasks" do+ it "enables maintenance and enqueues site-wide stats preparation" do+ Fabricate(:topic)+ max_topic_id = Topic.where(archetype: Archetype.default, deleted_at: nil).maximum(:id)+ SiteSetting.nested_replies_stats_maintenance_enabled = false++ output = capture_stdout { invoke_rake_task("nested_replies:prepare_stats") }++ expect(SiteSetting.nested_replies_stats_maintenance_enabled).to eq(true)+ expect(Jobs::PrepareNestedReplyStats.jobs.size).to eq(1)+ args = Jobs::PrepareNestedReplyStats.jobs.first["args"].first.with_indifferent_access+ expect(args).to include(max_topic_id: max_topic_id)+ expect(output).to include("Enqueued nested reply stats preparation")+ end+end