Skip to content

A platform for community discussion. Free, open, simple.

FIX: Run nested replies stats job on all topics when nested is default (#41510)

Mark VanLandingham971d1a28aa73
diff --git a/app/jobs/scheduled/backfill_nested_reply_stats.rb b/app/jobs/scheduled/backfill_nested_reply_stats.rb
index a3e133774d1..a74b3b0d6c8 100644
--- a/app/jobs/scheduled/backfill_nested_reply_stats.rb
+++ b/app/jobs/scheduled/backfill_nested_reply_stats.rb
@@ -26,19 +26,36 @@ module Jobs
<<~SQL,
SELECT t.id
FROM topics t
- INNER JOIN nested_topics nt ON nt.topic_id = t.id
+ LEFT JOIN nested_topics nt ON nt.topic_id = t.id
INNER JOIN posts op ON op.topic_id = t.id AND op.post_number = 1
LEFT JOIN nested_view_post_stats s ON s.post_id = op.id
WHERE t.deleted_at IS NULL
AND t.archetype = :archetype
+ AND (:nested_replies_default OR nt.topic_id IS NOT NULL)
#{category_filter}
- AND s.post_id IS NULL
+ AND (
+ s.post_id IS NULL
+ OR EXISTS (
+ SELECT 1
+ FROM posts child
+ INNER JOIN posts parent
+ ON parent.topic_id = child.topic_id
+ AND parent.post_number = child.reply_to_post_number
+ LEFT JOIN nested_view_post_stats parent_stats
+ ON parent_stats.post_id = parent.id
+ WHERE child.topic_id = t.id
+ AND child.reply_to_post_number IS NOT NULL
+ AND child.post_number > 1
+ AND parent_stats.post_id IS NULL
+ )
+ )
ORDER BY t.id DESC
LIMIT :batch_size
SQL
archetype: Archetype.default,
batch_size: SiteSetting.nested_replies_backfill_batch_size,
category_id: category_id,
+ nested_replies_default: SiteSetting.nested_replies_default,
)
end
diff --git a/app/services/nested_topic/list_children.rb b/app/services/nested_topic/list_children.rb
index e22053bda23..8570d7ad0fc 100644
--- a/app/services/nested_topic/list_children.rb
+++ b/app/services/nested_topic/list_children.rb
@@ -93,8 +93,9 @@ class NestedTopic::ListChildren
def prepare_posts(loader:, preloader:, all_posts:)
preloader.prepare(all_posts)
- context[:reply_counts] = loader.direct_reply_counts(all_posts.map(&:post_number))
- context[:descendant_counts] = loader.total_descendant_counts(all_posts.map(&:id))
+ counts = loader.tree_counts(all_posts)
+ context[:reply_counts] = counts[:reply_counts]
+ context[:descendant_counts] = counts[:descendant_counts]
end
def serialize_children(
diff --git a/app/services/nested_topic/list_roots.rb b/app/services/nested_topic/list_roots.rb
index 465031f557c..0cb775f3314 100644
--- a/app/services/nested_topic/list_roots.rb
+++ b/app/services/nested_topic/list_roots.rb
@@ -86,8 +86,9 @@ class NestedTopic::ListRoots
posts = params.page == 0 ? [loader.op_post] + all_posts : all_posts.dup
preloader.prepare(posts)
- context[:reply_counts] = loader.direct_reply_counts(posts.map(&:post_number))
- context[:descendant_counts] = loader.total_descendant_counts(posts.map(&:id))
+ counts = loader.tree_counts(posts)
+ context[:reply_counts] = counts[:reply_counts]
+ context[:descendant_counts] = counts[:descendant_counts]
end
def serialize_roots(
diff --git a/app/services/nested_topic/show_context.rb b/app/services/nested_topic/show_context.rb
index 8e1c44d3d62..0d62f84ac7e 100644
--- a/app/services/nested_topic/show_context.rb
+++ b/app/services/nested_topic/show_context.rb
@@ -128,8 +128,9 @@ class NestedTopic::ShowContext
all_posts.uniq!(&:id)
preloader.prepare(all_posts)
- context[:reply_counts] = loader.direct_reply_counts(all_posts.map(&:post_number))
- context[:descendant_counts] = loader.total_descendant_counts(all_posts.map(&:id))
+ counts = loader.tree_counts(all_posts)
+ context[:reply_counts] = counts[:reply_counts]
+ context[:descendant_counts] = counts[:descendant_counts]
end
def serialize_context(
diff --git a/lib/nested_replies/tree_loader.rb b/lib/nested_replies/tree_loader.rb
index 2c832cf2fe0..641a15fd89d 100644
--- a/lib/nested_replies/tree_loader.rb
+++ b/lib/nested_replies/tree_loader.rb
@@ -257,9 +257,88 @@ module NestedReplies
.count
end
- def total_descendant_counts(post_ids)
- return {} if post_ids.empty?
+ def tree_counts(posts)
+ reply_counts = direct_reply_counts(posts.map(&:post_number))
+ {
+ reply_counts: reply_counts,
+ descendant_counts: total_descendant_counts(posts, reply_counts: reply_counts),
+ }
+ end
+
+ def total_descendant_counts(posts, reply_counts: nil)
+ return {} if posts.empty?
+
+ post_records = posts.select { |post| post.respond_to?(:post_number) }
+ post_ids = posts.map { |post| post.respond_to?(:post_number) ? post.id : post }.compact.uniq
+
+ stat_counts = cached_total_descendant_counts(post_ids)
+
+ return stat_counts if post_records.empty?
+
+ reply_counts ||= {}
+ posts_needing_live_counts =
+ post_records.select do |post|
+ stat_counts[post.id].nil? ||
+ stat_counts[post.id].to_i < reply_counts[post.post_number].to_i
+ end
+
+ return stat_counts if posts_needing_live_counts.empty?
+
+ stat_counts.merge(
+ live_total_descendant_counts(posts_needing_live_counts),
+ ) { |_post_id, stat_count, live_count| [stat_count.to_i, live_count.to_i].max }
+ end
+
+ def live_total_descendant_counts(posts)
+ return {} if posts.empty?
+
+ post_ids_by_number = posts.index_by(&:post_number).transform_values(&:id)
+
+ rows =
+ DB.query(
+ <<~SQL,
+ WITH RECURSIVE descendants AS (
+ SELECT roots.post_number AS root_post_number,
+ child.post_number,
+ child.post_type,
+ 1 AS depth
+ FROM posts roots
+ JOIN posts child
+ ON child.topic_id = roots.topic_id
+ AND child.reply_to_post_number = roots.post_number
+ WHERE roots.topic_id = :topic_id
+ AND roots.id IN (:post_ids)
+ AND child.post_number > 1
+ UNION ALL
+ SELECT descendants.root_post_number,
+ child.post_number,
+ child.post_type,
+ descendants.depth + 1
+ FROM descendants
+ JOIN posts child
+ ON child.topic_id = :topic_id
+ AND child.reply_to_post_number = descendants.post_number
+ WHERE child.post_number > 1
+ AND descendants.depth < :max_cte_depth
+ )
+ SELECT root_post_number, COUNT(*) AS total_descendant_count
+ FROM descendants
+ WHERE post_type IN (:post_types)
+ GROUP BY root_post_number
+ SQL
+ topic_id: topic.id,
+ post_ids: post_ids_by_number.values,
+ post_types: visible_post_types,
+ max_cte_depth: 500,
+ )
+
+ rows.each_with_object({}) do |row, counts|
+ post_id = post_ids_by_number[row.root_post_number.to_i]
+ counts[post_id] = row.total_descendant_count.to_i if post_id
+ end
+ end
+ def cached_total_descendant_counts(post_ids)
if guardian.user&.whisperer?
NestedViewPostStat
.where(post_id: post_ids.uniq)
diff --git a/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rb b/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rb
index 831c3d576e9..4861111bddd 100644
--- a/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rb
+++ b/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rb
@@ -144,6 +144,18 @@ RSpec.describe Jobs::BackfillNestedReplyStats do
expect(NestedViewPostStat.find_by(post_id: non_nested_op.id)).to be_nil
end
+ it "processes topics without a record when nested replies are the default" do
+ SiteSetting.nested_replies_default = true
+ topic.nested_topic.destroy!
+ reply = Fabricate(:post, topic: topic, reply_to_post_number: 1)
+ Fabricate(:post, topic: topic, reply_to_post_number: reply.post_number)
+
+ NestedViewPostStat.delete_all
+ execute
+
+ expect(NestedViewPostStat.find_by(post_id: reply.id).direct_reply_count).to eq(1)
+ end
+
it "inserts a zero-count sentinel row for the OP of a topic with no replies" do
NestedViewPostStat.delete_all
@@ -190,6 +202,19 @@ RSpec.describe Jobs::BackfillNestedReplyStats do
expect(NestedViewPostStat.find_by(post_id: op.id).updated_at).to eq_time(initial_updated_at)
end
+ it "reprocesses topics with missing non-OP stats" do
+ parent = Fabricate(:post, topic: topic, reply_to_post_number: 1)
+ Fabricate(:post, topic: topic, reply_to_post_number: parent.post_number)
+
+ execute
+ NestedViewPostStat.where(post_id: parent.id).delete_all
+ execute
+
+ stat = NestedViewPostStat.find_by(post_id: parent.id)
+ expect(stat.direct_reply_count).to eq(1)
+ expect(stat.total_descendant_count).to eq(1)
+ end
+
it "limits backfill to the requested category" do
category = Fabricate(:category)
other_category = Fabricate(:category)
diff --git a/spec/requests/nested_topics_controller_spec.rb b/spec/requests/nested_topics_controller_spec.rb
index 50e4c8fafc3..8035fd6f892 100644
--- a/spec/requests/nested_topics_controller_spec.rb
+++ b/spec/requests/nested_topics_controller_spec.rb
@@ -1004,6 +1004,26 @@ RSpec.describe NestedTopicsController, type: :request do
expect(child_json["children"]).to eq([])
end
+ it "uses live descendant counts when stat rows are missing" do
+ SiteSetting.nested_replies_cap_nesting_depth = true
+ SiteSetting.nested_replies_max_depth = 3
+ parent = Fabricate(:post, topic: topic, user: user, reply_to_post_number: root.post_number)
+ child = Fabricate(:post, topic: topic, user: user, reply_to_post_number: parent.post_number)
+ Fabricate(:post, topic: topic, user: user, reply_to_post_number: child.post_number)
+ NestedViewPostStat.where(post_id: [parent.id, child.id]).delete_all
+ sign_in(user)
+
+ get children_url(topic, root.post_number, depth: 2)
+
+ json = response.parsed_body
+ parent_json = json["children"].find { |post_json| post_json["id"] == parent.id }
+ child_json = parent_json["children"].find { |post_json| post_json["id"] == child.id }
+ expect(parent_json["total_descendant_count"]).to eq(2)
+ expect(child_json["direct_reply_count"]).to eq(1)
+ expect(child_json["total_descendant_count"]).to eq(1)
+ expect(child_json["children"]).to eq([])
+ end
+
it "paginates flattened descendants inside the CTE" do
SiteSetting.nested_replies_cap_nesting_depth = true
SiteSetting.nested_replies_max_depth = 2