A platform for community discussion. Free, open, simple.
FIX: Run nested replies stats job on all topics when nested is default (#41510)
Mark VanLandingham971d1a28aa73
diff --git a/app/jobs/scheduled/backfill_nested_reply_stats.rb b/app/jobs/scheduled/backfill_nested_reply_stats.rbindex a3e133774d1..a74b3b0d6c8 100644--- a/app/jobs/scheduled/backfill_nested_reply_stats.rb+++ b/app/jobs/scheduled/backfill_nested_reply_stats.rb@@ -26,19 +26,36 @@ module Jobs<<~SQL,SELECT t.idFROM topics t- INNER JOIN nested_topics nt ON nt.topic_id = t.id+ LEFT JOIN nested_topics nt ON nt.topic_id = t.idINNER JOIN posts op ON op.topic_id = t.id AND op.post_number = 1LEFT JOIN nested_view_post_stats s ON s.post_id = op.idWHERE t.deleted_at IS NULLAND t.archetype = :archetype+ AND (:nested_replies_default OR nt.topic_id IS NOT NULL)#{category_filter}- AND s.post_id IS NULL+ AND (+ s.post_id IS NULL+ OR EXISTS (+ SELECT 1+ FROM posts child+ INNER JOIN posts parent+ ON parent.topic_id = child.topic_id+ AND parent.post_number = child.reply_to_post_number+ LEFT JOIN nested_view_post_stats parent_stats+ ON parent_stats.post_id = parent.id+ WHERE child.topic_id = t.id+ AND child.reply_to_post_number IS NOT NULL+ AND child.post_number > 1+ AND parent_stats.post_id IS NULL+ )+ )ORDER BY t.id DESCLIMIT :batch_sizeSQLarchetype: Archetype.default,batch_size: SiteSetting.nested_replies_backfill_batch_size,category_id: category_id,+ nested_replies_default: SiteSetting.nested_replies_default,)enddiff --git a/app/services/nested_topic/list_children.rb b/app/services/nested_topic/list_children.rbindex e22053bda23..8570d7ad0fc 100644--- a/app/services/nested_topic/list_children.rb+++ b/app/services/nested_topic/list_children.rb@@ -93,8 +93,9 @@ class NestedTopic::ListChildrendef prepare_posts(loader:, preloader:, all_posts:)preloader.prepare(all_posts)- context[:reply_counts] = loader.direct_reply_counts(all_posts.map(&:post_number))- context[:descendant_counts] = loader.total_descendant_counts(all_posts.map(&:id))+ counts = loader.tree_counts(all_posts)+ context[:reply_counts] = counts[:reply_counts]+ context[:descendant_counts] = counts[:descendant_counts]enddef serialize_children(diff --git a/app/services/nested_topic/list_roots.rb b/app/services/nested_topic/list_roots.rbindex 465031f557c..0cb775f3314 100644--- a/app/services/nested_topic/list_roots.rb+++ b/app/services/nested_topic/list_roots.rb@@ -86,8 +86,9 @@ class NestedTopic::ListRootsposts = params.page == 0 ? [loader.op_post] + all_posts : all_posts.duppreloader.prepare(posts)- context[:reply_counts] = loader.direct_reply_counts(posts.map(&:post_number))- context[:descendant_counts] = loader.total_descendant_counts(posts.map(&:id))+ counts = loader.tree_counts(posts)+ context[:reply_counts] = counts[:reply_counts]+ context[:descendant_counts] = counts[:descendant_counts]enddef serialize_roots(diff --git a/app/services/nested_topic/show_context.rb b/app/services/nested_topic/show_context.rbindex 8e1c44d3d62..0d62f84ac7e 100644--- a/app/services/nested_topic/show_context.rb+++ b/app/services/nested_topic/show_context.rb@@ -128,8 +128,9 @@ class NestedTopic::ShowContextall_posts.uniq!(&:id)preloader.prepare(all_posts)- context[:reply_counts] = loader.direct_reply_counts(all_posts.map(&:post_number))- context[:descendant_counts] = loader.total_descendant_counts(all_posts.map(&:id))+ counts = loader.tree_counts(all_posts)+ context[:reply_counts] = counts[:reply_counts]+ context[:descendant_counts] = counts[:descendant_counts]enddef serialize_context(diff --git a/lib/nested_replies/tree_loader.rb b/lib/nested_replies/tree_loader.rbindex 2c832cf2fe0..641a15fd89d 100644--- a/lib/nested_replies/tree_loader.rb+++ b/lib/nested_replies/tree_loader.rb@@ -257,9 +257,88 @@ module NestedReplies.countend- def total_descendant_counts(post_ids)- return {} if post_ids.empty?+ def tree_counts(posts)+ reply_counts = direct_reply_counts(posts.map(&:post_number))+ {+ reply_counts: reply_counts,+ descendant_counts: total_descendant_counts(posts, reply_counts: reply_counts),+ }+ end++ def total_descendant_counts(posts, reply_counts: nil)+ return {} if posts.empty?++ post_records = posts.select { |post| post.respond_to?(:post_number) }+ post_ids = posts.map { |post| post.respond_to?(:post_number) ? post.id : post }.compact.uniq++ stat_counts = cached_total_descendant_counts(post_ids)++ return stat_counts if post_records.empty?++ reply_counts ||= {}+ posts_needing_live_counts =+ post_records.select do |post|+ stat_counts[post.id].nil? ||+ stat_counts[post.id].to_i < reply_counts[post.post_number].to_i+ end++ return stat_counts if posts_needing_live_counts.empty?++ stat_counts.merge(+ live_total_descendant_counts(posts_needing_live_counts),+ ) { |_post_id, stat_count, live_count| [stat_count.to_i, live_count.to_i].max }+ end++ def live_total_descendant_counts(posts)+ return {} if posts.empty?++ post_ids_by_number = posts.index_by(&:post_number).transform_values(&:id)++ rows =+ DB.query(+ <<~SQL,+ WITH RECURSIVE descendants AS (+ SELECT roots.post_number AS root_post_number,+ child.post_number,+ child.post_type,+ 1 AS depth+ FROM posts roots+ JOIN posts child+ ON child.topic_id = roots.topic_id+ AND child.reply_to_post_number = roots.post_number+ WHERE roots.topic_id = :topic_id+ AND roots.id IN (:post_ids)+ AND child.post_number > 1+ UNION ALL+ SELECT descendants.root_post_number,+ child.post_number,+ child.post_type,+ descendants.depth + 1+ FROM descendants+ JOIN posts child+ ON child.topic_id = :topic_id+ AND child.reply_to_post_number = descendants.post_number+ WHERE child.post_number > 1+ AND descendants.depth < :max_cte_depth+ )+ SELECT root_post_number, COUNT(*) AS total_descendant_count+ FROM descendants+ WHERE post_type IN (:post_types)+ GROUP BY root_post_number+ SQL+ topic_id: topic.id,+ post_ids: post_ids_by_number.values,+ post_types: visible_post_types,+ max_cte_depth: 500,+ )++ rows.each_with_object({}) do |row, counts|+ post_id = post_ids_by_number[row.root_post_number.to_i]+ counts[post_id] = row.total_descendant_count.to_i if post_id+ end+ end+ def cached_total_descendant_counts(post_ids)if guardian.user&.whisperer?NestedViewPostStat.where(post_id: post_ids.uniq)diff --git a/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rb b/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rbindex 831c3d576e9..4861111bddd 100644--- a/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rb+++ b/spec/jobs/scheduled/backfill_nested_reply_stats_spec.rb@@ -144,6 +144,18 @@ RSpec.describe Jobs::BackfillNestedReplyStats doexpect(NestedViewPostStat.find_by(post_id: non_nested_op.id)).to be_nilend+ it "processes topics without a record when nested replies are the default" do+ SiteSetting.nested_replies_default = true+ topic.nested_topic.destroy!+ reply = Fabricate(:post, topic: topic, reply_to_post_number: 1)+ Fabricate(:post, topic: topic, reply_to_post_number: reply.post_number)++ NestedViewPostStat.delete_all+ execute++ expect(NestedViewPostStat.find_by(post_id: reply.id).direct_reply_count).to eq(1)+ end+it "inserts a zero-count sentinel row for the OP of a topic with no replies" doNestedViewPostStat.delete_all@@ -190,6 +202,19 @@ RSpec.describe Jobs::BackfillNestedReplyStats doexpect(NestedViewPostStat.find_by(post_id: op.id).updated_at).to eq_time(initial_updated_at)end+ it "reprocesses topics with missing non-OP stats" do+ parent = Fabricate(:post, topic: topic, reply_to_post_number: 1)+ Fabricate(:post, topic: topic, reply_to_post_number: parent.post_number)++ execute+ NestedViewPostStat.where(post_id: parent.id).delete_all+ execute++ stat = NestedViewPostStat.find_by(post_id: parent.id)+ expect(stat.direct_reply_count).to eq(1)+ expect(stat.total_descendant_count).to eq(1)+ end+it "limits backfill to the requested category" docategory = Fabricate(:category)other_category = Fabricate(:category)diff --git a/spec/requests/nested_topics_controller_spec.rb b/spec/requests/nested_topics_controller_spec.rbindex 50e4c8fafc3..8035fd6f892 100644--- a/spec/requests/nested_topics_controller_spec.rb+++ b/spec/requests/nested_topics_controller_spec.rb@@ -1004,6 +1004,26 @@ RSpec.describe NestedTopicsController, type: :request doexpect(child_json["children"]).to eq([])end+ it "uses live descendant counts when stat rows are missing" do+ SiteSetting.nested_replies_cap_nesting_depth = true+ SiteSetting.nested_replies_max_depth = 3+ parent = Fabricate(:post, topic: topic, user: user, reply_to_post_number: root.post_number)+ child = Fabricate(:post, topic: topic, user: user, reply_to_post_number: parent.post_number)+ Fabricate(:post, topic: topic, user: user, reply_to_post_number: child.post_number)+ NestedViewPostStat.where(post_id: [parent.id, child.id]).delete_all+ sign_in(user)++ get children_url(topic, root.post_number, depth: 2)++ json = response.parsed_body+ parent_json = json["children"].find { |post_json| post_json["id"] == parent.id }+ child_json = parent_json["children"].find { |post_json| post_json["id"] == child.id }+ expect(parent_json["total_descendant_count"]).to eq(2)+ expect(child_json["direct_reply_count"]).to eq(1)+ expect(child_json["total_descendant_count"]).to eq(1)+ expect(child_json["children"]).to eq([])+ end+it "paginates flattened descendants inside the CTE" doSiteSetting.nested_replies_cap_nesting_depth = trueSiteSetting.nested_replies_max_depth = 2