Skip to content

A platform for community discussion. Free, open, simple.

PERF: Add fast nested reply stats preparation (#41870)

Mark VanLandinghamc85593eda88d
diff --git a/app/jobs/regular/prepare_nested_reply_stats.rb b/app/jobs/regular/prepare_nested_reply_stats.rb
new file mode 100644
index 00000000000..bf603d58451
--- /dev/null
+++ b/app/jobs/regular/prepare_nested_reply_stats.rb
@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Jobs
+ class PrepareNestedReplyStats < ::Jobs::Base
+ sidekiq_options queue: "low"
+ cluster_concurrency 1
+
+ def execute(args = {})
+ raise Discourse::ReadOnly if Discourse.readonly_mode?
+
+ args ||= {}
+ isolated_topic_id = args[:topic_id].to_i
+ if isolated_topic_id.positive?
+ Jobs::BackfillNestedReplyStats.backfill_topic(isolated_topic_id)
+ return
+ end
+
+ after_topic_id = args[:after_topic_id].to_i
+ max_topic_id = args[:max_topic_id].to_i
+ max_topic_id = maximum_topic_id if max_topic_id.zero?
+
+ if after_topic_id >= max_topic_id
+ log_completion(max_topic_id)
+ return
+ end
+
+ topic_ids = topic_ids_to_prepare(after_topic_id:, max_topic_id:)
+ if topic_ids.empty?
+ log_completion(max_topic_id)
+ return
+ end
+
+ topic_ids.each do |topic_id|
+ Jobs::BackfillNestedReplyStats.backfill_topic(topic_id)
+ rescue => error
+ Discourse.warn_exception(
+ error,
+ message: "Failed to prepare nested reply stats for topic #{topic_id}",
+ )
+ Jobs.enqueue(:prepare_nested_reply_stats, topic_id: topic_id)
+ end
+ after_topic_id = topic_ids.last
+
+ if after_topic_id < max_topic_id
+ Jobs.enqueue(:prepare_nested_reply_stats, after_topic_id:, max_topic_id:)
+ else
+ log_completion(max_topic_id)
+ end
+ end
+
+ private
+
+ def maximum_topic_id
+ Topic.where(archetype: Archetype.default, deleted_at: nil).maximum(:id).to_i
+ end
+
+ def topic_ids_to_prepare(after_topic_id:, max_topic_id:)
+ DB.query_single(
+ <<~SQL,
+ SELECT topics.id
+ FROM topics
+ INNER JOIN posts op
+ ON op.topic_id = topics.id
+ AND op.post_number = 1
+ WHERE topics.id > :after_topic_id
+ AND topics.id <= :max_topic_id
+ AND topics.deleted_at IS NULL
+ AND topics.archetype = :archetype
+ ORDER BY topics.id
+ LIMIT :batch_size
+ SQL
+ after_topic_id: after_topic_id,
+ max_topic_id: max_topic_id,
+ archetype: Archetype.default,
+ batch_size: SiteSetting.nested_replies_backfill_batch_size,
+ )
+ end
+
+ def log_completion(max_topic_id)
+ Rails.logger.info("Nested reply stats preparation completed through topic #{max_topic_id}")
+ end
+ end
+end
diff --git a/app/jobs/scheduled/backfill_nested_reply_stats.rb b/app/jobs/scheduled/backfill_nested_reply_stats.rb
index a74b3b0d6c8..abddc13b0d4 100644
--- a/app/jobs/scheduled/backfill_nested_reply_stats.rb
+++ b/app/jobs/scheduled/backfill_nested_reply_stats.rb
@@ -13,7 +13,7 @@ module Jobs
topic_ids = topic_ids_missing_stats(category_id: args[:category_id])
return if topic_ids.empty?
- topic_ids.each { |topic_id| backfill_topic(topic_id) }
+ topic_ids.each { |topic_id| self.class.backfill_topic(topic_id) }
end
private
@@ -59,7 +59,7 @@ module Jobs
)
end
- def backfill_topic(topic_id)
+ def self.backfill_topic(topic_id)
DB.exec(<<~SQL, topic_id: topic_id, whisper_type: Post.types[:whisper])
WITH RECURSIVE
edges AS (
diff --git a/app/models/concerns/has_nested_reply_stats.rb b/app/models/concerns/has_nested_reply_stats.rb
index 0cf50123513..63ea8e7fe39 100644
--- a/app/models/concerns/has_nested_reply_stats.rb
+++ b/app/models/concerns/has_nested_reply_stats.rb
@@ -13,7 +13,7 @@ module HasNestedReplyStats
# are left alone — their total descendant count is unchanged — while
# ancestors unique to either chain are adjusted by this post's subtree size.
def nested_replies_apply_reparent(previous_reply_to_post_number)
- return unless SiteSetting.nested_replies_enabled
+ return unless maintain_nested_reply_stats?
return if previous_reply_to_post_number == reply_to_post_number
subtree_size, whisper_subtree_size = nested_replies_subtree_sizes
@@ -76,8 +76,12 @@ module HasNestedReplyStats
private
+ def maintain_nested_reply_stats?
+ SiteSetting.nested_replies_enabled || SiteSetting.nested_replies_stats_maintenance_enabled
+ end
+
def nested_replies_increment_stats
- return unless SiteSetting.nested_replies_enabled
+ return unless maintain_nested_reply_stats?
return if reply_to_post_number.blank?
ancestors = nested_replies_walk(reply_to_post_number)
@@ -110,7 +114,7 @@ module HasNestedReplyStats
end
def nested_replies_decrement_stats
- return unless SiteSetting.nested_replies_enabled
+ return unless maintain_nested_reply_stats?
if reply_to_post_number.present?
subtree_size, whisper_subtree_size = nested_replies_subtree_sizes
is_whisper = post_type == Post.types[:whisper] ? 1 : 0
diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml
index d99ef1b2677..7e5b491d6e1 100644
--- a/config/locales/server.en.yml
+++ b/config/locales/server.en.yml
@@ -2951,7 +2951,8 @@ en:
nested_replies_hot_preload_depth_penalty: "Priority subtracted for each level of depth when choosing nested replies to pre-expand in hot view."
nested_replies_default_sort: "Default sort algorithm for nested replies view."
nested_replies_max_depth: "Maximum visual nesting depth for threaded replies. Posts deeper than this show a 'Continue this thread' link."
- nested_replies_backfill_batch_size: "Number of topics to backfill nested reply stats for per scheduled run."
+ nested_replies_backfill_batch_size: "Number of topics to process per nested reply stats backfill batch."
+ nested_replies_stats_maintenance_enabled: "Maintain nested reply statistics even while nested replies are disabled."
nested_replies_cap_nesting_depth: "Cap reply nesting at the maximum visual depth. When enabled, replies to the deepest level are re-parented as siblings instead of children."
enable_new_defaults_for_trust_level_3_requirements: "We have updated the TL3 default requirements so they’re more reasonable for members of large, active communities. If you have previously overridden these settings, this upcoming change will have no effect."
floating_dismiss_topics_on_mobile: "Shows a floating 'Dismiss...' button on mobile for the topic list for easier access and to free up space in the top of the topic list"
diff --git a/config/site_settings.yml b/config/site_settings.yml
index 5c0cff0f191..62eb3b38450 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -2119,6 +2119,9 @@ nested_replies:
min: 1
max: 10000
hidden: true
+ nested_replies_stats_maintenance_enabled:
+ default: false
+ hidden: true
email:
email_time_window_mins:
diff --git a/lib/tasks/nested_replies.rake b/lib/tasks/nested_replies.rake
new file mode 100644
index 00000000000..fb412b7b0ed
--- /dev/null
+++ b/lib/tasks/nested_replies.rake
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+namespace :nested_replies do
+ desc "Prepare nested reply statistics for all regular topics"
+ task prepare_stats: :environment do
+ enqueue =
+ lambda do
+ database = RailsMultisite::ConnectionManagement.current_db
+ SiteSetting.nested_replies_stats_maintenance_enabled = true
+ max_topic_id = Topic.where(archetype: Archetype.default, deleted_at: nil).maximum(:id).to_i
+ Jobs.enqueue(:prepare_nested_reply_stats, max_topic_id: max_topic_id)
+
+ puts "Enqueued nested reply stats preparation for '#{database}' " \
+ "through topic #{max_topic_id}"
+ end
+
+ if ENV["RAILS_DB"]
+ enqueue.call
+ else
+ RailsMultisite::ConnectionManagement.each_connection { enqueue.call }
+ end
+ end
+end
diff --git a/spec/jobs/prepare_nested_reply_stats_spec.rb b/spec/jobs/prepare_nested_reply_stats_spec.rb
new file mode 100644
index 00000000000..782b5826956
--- /dev/null
+++ b/spec/jobs/prepare_nested_reply_stats_spec.rb
@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+RSpec.describe Jobs::PrepareNestedReplyStats do
+ before do
+ SiteSetting.nested_replies_enabled = false
+ SiteSetting.nested_replies_default = false
+ SiteSetting.nested_replies_stats_maintenance_enabled = false
+ end
+
+ it "prepares every regular topic while nested replies remain disabled" do
+ topic = Fabricate(:topic)
+ op = Fabricate(:post, topic: topic, post_number: 1)
+ Fabricate(:post, topic: topic, reply_to_post_number: op.post_number)
+
+ deleted_topic = Fabricate(:topic, deleted_at: Time.current)
+ deleted_op = Fabricate(:post, topic: deleted_topic, post_number: 1)
+
+ private_message = Fabricate(:private_message_topic)
+ private_message_op = Fabricate(:post, topic: private_message, post_number: 1)
+
+ described_class.new.execute(max_topic_id: Topic.maximum(:id))
+
+ expect(NestedViewPostStat.find_by(post_id: op.id)).to have_attributes(
+ direct_reply_count: 1,
+ total_descendant_count: 1,
+ )
+ expect(NestedViewPostStat.exists?(post_id: deleted_op.id)).to eq(false)
+ expect(NestedViewPostStat.exists?(post_id: private_message_op.id)).to eq(false)
+ end
+
+ it "immediately continues from the last topic in each bounded batch" do
+ after_topic_id = Topic.maximum(:id).to_i
+ first_topic = Fabricate(:topic)
+ first_op = Fabricate(:post, topic: first_topic, post_number: 1)
+ second_topic = Fabricate(:topic)
+ second_op = Fabricate(:post, topic: second_topic, post_number: 1)
+ max_topic_id = second_topic.id
+ later_topic = Fabricate(:topic)
+ later_op = Fabricate(:post, topic: later_topic, post_number: 1)
+ SiteSetting.nested_replies_backfill_batch_size = 1
+
+ described_class.new.execute(after_topic_id:, max_topic_id:)
+
+ expect(NestedViewPostStat.exists?(post_id: first_op.id)).to eq(true)
+ expect(NestedViewPostStat.exists?(post_id: second_op.id)).to eq(false)
+ expect(described_class.jobs.size).to eq(1)
+ expect(described_class.jobs.first["at"]).to be_nil
+ continuation = described_class.jobs.first["args"].first.with_indifferent_access
+ expect(continuation).to include(after_topic_id: first_topic.id, max_topic_id: max_topic_id)
+
+ described_class.jobs.clear
+ described_class.new.execute(continuation)
+
+ expect(NestedViewPostStat.exists?(post_id: second_op.id)).to eq(true)
+ expect(NestedViewPostStat.exists?(post_id: later_op.id)).to eq(false)
+ expect(described_class.jobs).to be_empty
+ end
+
+ it "isolates a failed topic so later topics are still prepared" do
+ after_topic_id = Topic.maximum(:id).to_i
+ failed_topic = Fabricate(:topic)
+ Fabricate(:post, topic: failed_topic, post_number: 1)
+ healthy_topic = Fabricate(:topic)
+ healthy_op = Fabricate(:post, topic: healthy_topic, post_number: 1)
+ allow(Discourse).to receive(:warn_exception)
+ allow(Jobs::BackfillNestedReplyStats).to receive(:backfill_topic).and_call_original
+ allow(Jobs::BackfillNestedReplyStats).to receive(:backfill_topic).with(
+ failed_topic.id,
+ ).and_raise("boom")
+
+ described_class.new.execute(after_topic_id:, max_topic_id: healthy_topic.id)
+
+ expect(NestedViewPostStat.exists?(post_id: healthy_op.id)).to eq(true)
+ expect(described_class.jobs.size).to eq(1)
+ retry_args = described_class.jobs.first["args"].first.with_indifferent_access
+ expect(retry_args).to include(topic_id: failed_topic.id)
+ expect { described_class.new.execute(retry_args) }.to raise_error("boom")
+ end
+end
diff --git a/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rb b/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rb
index 4861111bddd..27b414701cd 100644
--- a/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rb
+++ b/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rb
@@ -11,9 +11,10 @@ RSpec.describe Jobs::BackfillNestedReplyStats do
described_class.new.execute(args)
end
- it "does nothing when feature is disabled" do
+ it "does nothing when feature is disabled, even during explicit preparation" do
SiteSetting.nested_replies_enabled = false
Fabricate(:post, topic: topic, reply_to_post_number: 1)
+ SiteSetting.nested_replies_stats_maintenance_enabled = true
execute
diff --git a/spec/lib/nested_replies/nested_replies_n_plus_one_spec.rb b/spec/lib/nested_replies/nested_replies_n_plus_one_spec.rb
index 0eaa57bfdd0..23c91f39c1a 100644
--- a/spec/lib/nested_replies/nested_replies_n_plus_one_spec.rb
+++ b/spec/lib/nested_replies/nested_replies_n_plus_one_spec.rb
@@ -32,6 +32,23 @@ RSpec.describe "Nested replies N+1 elimination", type: :request do
end
describe "after_create stats increment" do
+ it "maintains stats during preparation while nested replies are disabled" do
+ SiteSetting.nested_replies_enabled = false
+ SiteSetting.nested_replies_stats_maintenance_enabled = true
+
+ parent = Fabricate(:post, topic: topic, user: user, reply_to_post_number: op.post_number)
+ Fabricate(:post, topic: topic, user: user, reply_to_post_number: parent.post_number)
+
+ expect(NestedViewPostStat.find_by(post_id: op.id)).to have_attributes(
+ direct_reply_count: 1,
+ total_descendant_count: 2,
+ )
+ expect(NestedViewPostStat.find_by(post_id: parent.id)).to have_attributes(
+ direct_reply_count: 1,
+ total_descendant_count: 1,
+ )
+ end
+
it "uses constant queries regardless of chain depth" do
chain_3 = create_reply_chain(depth: 3)
queries_3 =
diff --git a/spec/tasks/nested_replies_spec.rb b/spec/tasks/nested_replies_spec.rb
new file mode 100644
index 00000000000..08525797e5c
--- /dev/null
+++ b/spec/tasks/nested_replies_spec.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+RSpec.describe "nested_replies tasks" do
+ it "enables maintenance and enqueues site-wide stats preparation" do
+ Fabricate(:topic)
+ max_topic_id = Topic.where(archetype: Archetype.default, deleted_at: nil).maximum(:id)
+ SiteSetting.nested_replies_stats_maintenance_enabled = false
+
+ output = capture_stdout { invoke_rake_task("nested_replies:prepare_stats") }
+
+ expect(SiteSetting.nested_replies_stats_maintenance_enabled).to eq(true)
+ expect(Jobs::PrepareNestedReplyStats.jobs.size).to eq(1)
+ args = Jobs::PrepareNestedReplyStats.jobs.first["args"].first.with_indifferent_access
+ expect(args).to include(max_topic_id: max_topic_id)
+ expect(output).to include("Enqueued nested reply stats preparation")
+ end
+end