A platform for community discussion. Free, open, simple.
FIX: Only count public posts in Rewind Top Words and FBFF
Régis Hanolc2835fef529a
diff --git a/plugins/discourse-rewind/app/services/discourse_rewind/action/base_report.rb b/plugins/discourse-rewind/app/services/discourse_rewind/action/base_report.rbindex c796811d33e..193e3933e33 100644--- a/plugins/discourse-rewind/app/services/discourse_rewind/action/base_report.rb+++ b/plugins/discourse-rewind/app/services/discourse_rewind/action/base_report.rb@@ -10,6 +10,14 @@ module DiscourseRewindraise NotImplementedErrorend+ def self.publicly_visible_topics+ Topic.listable_topics.visible.secured+ end++ def self.publicly_visible_posts+ Post.visible.merge(publicly_visible_topics).where.not(post_type: Post.types[:whisper])+ end+def self.enabled?trueenddiff --git a/plugins/discourse-rewind/app/services/discourse_rewind/action/fbff.rb b/plugins/discourse-rewind/app/services/discourse_rewind/action/fbff.rbindex 77ebb4193ee..c23045f7153 100644--- a/plugins/discourse-rewind/app/services/discourse_rewind/action/fbff.rb+++ b/plugins/discourse-rewind/app/services/discourse_rewind/action/fbff.rb@@ -1,7 +1,5 @@# frozen_string_literal: true-# Forum Best Friend Forever ranking-# Score is informative only, do not show in UImodule DiscourseRewindmodule Actionclass Fbff < BaseReport@@ -30,63 +28,23 @@ module DiscourseRewinddef callreturn FakeData if should_use_fake_data?- most_liked_users =- like_query(user, date)- .where(acting_user_id: user.id)- .group(:user_id)+ scores = Hash.new(0)+ [+ [like_query.where(acting_user_id: user.id), :user_id, LIKE_SCORE],+ [like_query.where(user_id: user.id), :acting_user_id, LIKE_SCORE],+ [post_query.where(posts: { user_id: user.id }), "replies.user_id", REPLY_SCORE],+ [post_query.where(replies: { user_id: user.id }), "posts.user_id", REPLY_SCORE],+ ].each do |query, friend_column, weight|+ query+ .where(friend_column => eligible_users)+ .group(friend_column).order("COUNT(*) DESC").limit(MAX_SUMMARY_RESULTS)- .pluck("user_actions.user_id, COUNT(*)")- .map { |user_id, count| { user_id => count } }- .reduce({}, :merge)-- most_liked_by_users =- like_query(user, date)- .where(user: user)- .group(:acting_user_id)- .order("COUNT(*) DESC")- .limit(MAX_SUMMARY_RESULTS)- .pluck("acting_user_id, COUNT(*)")- .map { |acting_user_id, count| { acting_user_id => count } }- .reduce({}, :merge)-- users_who_most_replied_me =- post_query(user, date)- .where(posts: { user_id: user.id })- .group("replies.user_id")- .order("COUNT(*) DESC")- .limit(MAX_SUMMARY_RESULTS)- .pluck("replies.user_id, COUNT(*)")- .map { |user_id, count| { user_id => count } }- .reduce({}, :merge)-- users_i_most_replied =- post_query(user, date)- .where("replies.user_id = ?", user.id)- .group("posts.user_id")- .order("COUNT(*) DESC")- .limit(MAX_SUMMARY_RESULTS)- .pluck("posts.user_id, COUNT(*)")- .map { |user_id, count| { user_id => count } }- .reduce({}, :merge)-- # NOTE: At some point maybe we want to include chat interactions- # in the calculations here.- fbffs = [- apply_score(most_liked_users, LIKE_SCORE),- apply_score(most_liked_by_users, LIKE_SCORE),- apply_score(users_who_most_replied_me, REPLY_SCORE),- apply_score(users_i_most_replied, REPLY_SCORE),- ]-- fbff_id =- fbffs- .flatten- .inject { |h1, h2| h1.merge(h2) { |_, v1, v2| v1 + v2 } }- &.sort_by { |_, v| -v }- &.first- &.first+ .count+ .each { |user_id, count| scores[user_id] += count * weight }+ end+ fbff_id = scores.max_by(&:last)&.firstreturn if !fbff_id{@@ -98,66 +56,43 @@ module DiscourseRewind}end- def post_query(user, date)- Post- .with(eligible_users: User.real.activated.not_suspended.select(:id))- .joins(:topic)- .includes(:topic)- .where(- "posts.post_type IN (?)",- Topic.visible_post_types(user, include_moderator_actions: false),- )+ def post_query+ self+ .class+ .publicly_visible_posts.joins(- "INNER JOIN posts replies ON posts.topic_id = replies.topic_id AND posts.reply_to_post_number = replies.post_number",+ "INNER JOIN posts replies ON replies.topic_id = posts.topic_id AND replies.post_number = posts.reply_to_post_number",)- .joins(- "INNER JOIN topics ON replies.topic_id = topics.id- AND topics.archetype <> 'private_message'- AND replies.post_type IN (#{Topic.visible_post_types(user, include_moderator_actions: false).join(",")})",+ .where(posts: { post_type: Post.types[:regular], created_at: date })+ .where(+ replies: {+ post_type: Post.types[:regular],+ hidden: false,+ created_at: date,+ deleted_at: nil,+ },)- .joins("INNER JOIN eligible_users eu ON eu.id = replies.user_id")- .joins("INNER JOIN eligible_users eu2 ON eu2.id = posts.user_id")- .where("replies.created_at BETWEEN ? AND ?", date.first, date.last)- .where("posts.created_at BETWEEN ? AND ?", date.first, date.last).where("replies.user_id <> posts.user_id")- .where(<<~SQL, user_id: user.id)- NOT EXISTS (- SELECT 1 FROM muted_users- WHERE muted_users.user_id = :user_id- AND muted_users.muted_user_id IN (replies.user_id, posts.user_id)- )- AND NOT EXISTS (- SELECT 1 FROM ignored_users- WHERE ignored_users.user_id = :user_id- AND ignored_users.ignored_user_id IN (replies.user_id, posts.user_id)- )- SQLend- def like_query(user, date)+ def like_queryUserAction- .with(eligible_users: User.real.activated.not_suspended.select(:id)).joins(:target_topic, :target_post)- .joins("INNER JOIN eligible_users eu ON eu.id = user_actions.user_id")- .joins("INNER JOIN eligible_users eu2 ON eu2.id = user_actions.acting_user_id")- .where(created_at: date)- .where(action_type: UserAction::WAS_LIKED)- .where(<<~SQL, user_id: user.id)- NOT EXISTS (- SELECT 1 FROM muted_users- WHERE muted_users.user_id = :user_id- AND muted_users.muted_user_id IN (user_actions.user_id, user_actions.acting_user_id)- )- AND NOT EXISTS (- SELECT 1 FROM ignored_users- WHERE ignored_users.user_id = :user_id- AND ignored_users.ignored_user_id IN (user_actions.user_id, user_actions.acting_user_id)- )- SQL+ .merge(self.class.publicly_visible_topics)+ .where(action_type: UserAction::WAS_LIKED, created_at: date)+ .where(posts: { post_type: Post.types[:regular], hidden: false })end- def apply_score(users, score)- users.map { |user_id, count| { user_id => count * score } }+ private++ def eligible_users+ @eligible_users ||=+ User+ .real+ .activated+ .not_suspended+ .where.not(id: user.muted_user_ids | user.ignored_user_ids)+ .select(:id)endendenddiff --git a/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rb b/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rbindex 97f42544666..fc8736e5c37 100644--- a/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rb+++ b/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rb@@ -47,22 +47,16 @@ module DiscourseRewindenddef word_query- DB.query(<<~SQL, user_id: user.id, date_start: date.first, date_end: date.last)+ posts = self.class.publicly_visible_posts.where(user_id: user.id, created_at: date)++ DB.query(<<~SQL)WITH popular_words AS (SELECT*FROMts_stat($INNERSQL$- SELECT- search_data- FROM- post_search_data- INNER JOIN- posts ON posts.id = post_search_data.post_id- WHERE- posts.user_id = :user_id- AND posts.created_at BETWEEN :date_start AND :date_end+ #{posts.joins(:post_search_data).select(:search_data).to_sql}$INNERSQL$) AS search_dataWHERE LENGTH(word) >= 2@@ -84,15 +78,7 @@ module DiscourseRewindword as original_wordFROMts_stat ($INNERSQL$- SELECT- to_tsvector('simple',- regexp_replace(raw, 'https?://[^\\s]+', ' ', 'g')- )- FROM- posts AS p- WHERE- p.user_id = :user_id- AND p.created_at BETWEEN :date_start AND :date_end+ #{posts.select("to_tsvector('simple', regexp_replace(raw, 'https?://[^\\s]+', ' ', 'g'))").to_sql}$INNERSQL$)WHERE LENGTH(word) >= 2AND word ~ '^[a-zA-Z]+$'diff --git a/plugins/discourse-rewind/lib/discourse_rewind/fetch_reports_helper.rb b/plugins/discourse-rewind/lib/discourse_rewind/fetch_reports_helper.rbindex 84fb3ee2d92..2d401c13857 100644--- a/plugins/discourse-rewind/lib/discourse_rewind/fetch_reports_helper.rb+++ b/plugins/discourse-rewind/lib/discourse_rewind/fetch_reports_helper.rb@@ -4,7 +4,7 @@ module DiscourseRewindmodule FetchReportsHelperextend self- REWIND_REPORT_VERSION = "2"+ REWIND_REPORT_VERSION = "3"CACHE_DURATION = Rails.env.development? ? 10.seconds : 3.daysdef cache_key(user_id, year)diff --git a/plugins/discourse-rewind/spec/actions/base_report_spec.rb b/plugins/discourse-rewind/spec/actions/base_report_spec.rbnew file mode 100644index 00000000000..8bd4386e73f--- /dev/null+++ b/plugins/discourse-rewind/spec/actions/base_report_spec.rb@@ -0,0 +1,35 @@+# frozen_string_literal: true++RSpec.describe DiscourseRewind::Action::BaseReport do+ fab!(:public_topic, :topic)+ fab!(:unlisted_topic) { Fabricate(:topic, visible: false) }+ fab!(:private_message_topic)+ fab!(:restricted_topic) do+ Fabricate(:topic, category: Fabricate(:private_category, group: Fabricate(:group)))+ end+ fab!(:topics) { [public_topic, unlisted_topic, private_message_topic, restricted_topic] }++ describe ".publicly_visible_topics" do+ it "only returns listed topics in public categories" do+ expect(described_class.publicly_visible_topics.where(id: topics)).to contain_exactly(+ public_topic,+ )+ end+ end++ describe ".publicly_visible_posts" do+ it "only returns visible, non-whisper posts in publicly visible topics" do+ public_post = Fabricate(:post, topic: public_topic)+ posts = [+ public_post,+ Fabricate(:post, topic: public_topic, hidden: true),+ Fabricate(:post, topic: public_topic, post_type: Post.types[:whisper]),+ *topics.drop(1).map { |topic| Fabricate(:post, topic:) },+ ]++ expect(described_class.publicly_visible_posts.where(id: posts)).to contain_exactly(+ public_post,+ )+ end+ end+enddiff --git a/plugins/discourse-rewind/spec/actions/fbff_spec.rb b/plugins/discourse-rewind/spec/actions/fbff_spec.rbindex 2259528bc07..8f837ce5b42 100644--- a/plugins/discourse-rewind/spec/actions/fbff_spec.rb+++ b/plugins/discourse-rewind/spec/actions/fbff_spec.rb@@ -1,107 +1,173 @@# frozen_string_literal: trueRSpec.describe DiscourseRewind::Action::Fbff do- fab!(:date) { Date.new(2021).all_year }fab!(:user)fab!(:other_user, :user)- fab!(:bot)-fab!(:topic)- fab!(:post_1) { Fabricate(:post, topic: topic, user: user, created_at: random_datetime) }+ fab!(:post_1) { Fabricate(:post, topic:, user:, created_at: random_datetime) }fab!(:post_2) doFabricate(:post,- topic: topic,+ topic:,user: other_user,created_at: random_datetime,- reply_to_post_number: 1,+ reply_to_post_number: post_1.post_number,)endfab!(:post_3) doFabricate(:post,- topic: topic,- user: user,+ topic:,+ user:,created_at: random_datetime,reply_to_post_number: post_2.post_number,)end+ fab!(:restricted_category) { Fabricate(:private_category, group: Fabricate(:group)) }++ describe ".call" do+ it "ranks a reply higher than a like" do+ liker = Fabricate(:user)+ 9.times do+ liked_post = Fabricate(:post, user:, created_at: random_datetime)+ Fabricate(+ :user_action,+ action_type: UserAction::WAS_LIKED,+ acting_user: liker,+ user:,+ target_post: liked_post,+ target_topic: liked_post.topic,+ created_at: random_datetime,+ )+ end- describe ".post_query" do- it "includes the correct posts for the time period based on replies between users" do- expect(- described_class.new(user: user, date: date).post_query(user, date).map(&:id),- ).to match_array([post_2.id, post_3.id])+ expect(call_report[:data][:fbff][:id]).to eq(other_user.id)end- it "excludes posts by bot users" do- post_2.update!(user: bot)+ context "when there are only likes" do+ fab!(:friend, :user)- expect(- described_class.new(user: user, date: date).post_query(user, date).map(&:id),- ).not_to include(post_2.id)+ before { [post_2, post_3].each(&:trash!) }++ def like(acting_user:, liked_user:)+ liked_post = Fabricate(:post, user: liked_user, created_at: random_datetime)+ Fabricate(+ :user_action,+ action_type: UserAction::WAS_LIKED,+ acting_user:,+ user: liked_user,+ target_post: liked_post,+ target_topic: liked_post.topic,+ created_at: random_datetime,+ )+ end++ it "counts likes the user gave" do+ like(acting_user: user, liked_user: friend)++ expect(call_report[:data][:fbff][:id]).to eq(friend.id)+ end++ it "counts likes the user received" do+ like(acting_user: friend, liked_user: user)++ expect(call_report[:data][:fbff][:id]).to eq(friend.id)+ endend- it "excludes posts by deactivated users" do- user.deactivate(Discourse.system_user)+ context "when the only friend exchanges likes and replies with the user" do+ before do+ [[other_user, user, post_1], [user, other_user, post_2]].each do |acting_user, liked, post|+ Fabricate(+ :user_action,+ action_type: UserAction::WAS_LIKED,+ acting_user:,+ user: liked,+ target_post: post,+ target_topic: topic,+ created_at: random_datetime,+ )+ end+ end++ it "ignores them when they are not active" do+ other_user.deactivate(Discourse.system_user)- expect(- described_class.new(user: user, date: date).post_query(user, date).map(&:id),- ).not_to include(post_2.id)+ expect(call_report).to be_nil+ end++ it "ignores them when they are muted" do+ Fabricate(:muted_user, user:, muted_user: other_user)++ expect(call_report).to be_nil+ end++ it "ignores them when they are ignored" do+ Fabricate(:ignored_user, user:, ignored_user: other_user)++ expect(call_report).to be_nil+ endend+ end- it "excludes posts by suspended users" do- user.suspended_at = Time.now- user.suspended_till = 200.years.from_now- user.save!+ describe "#post_query" do+ subject(:post_ids) { described_class.new(user:, date:).post_query.map(&:id) }- expect(- described_class.new(user: user, date: date).post_query(user, date).map(&:id),- ).not_to include(post_2.id)+ it "returns the replies between users" do+ expect(post_ids).to contain_exactly(post_2.id, post_3.id)end- it "excludes moderator action posts" do- post_2.update!(post_type: Post.types[:moderator_action])+ it "excludes replies to the user's own posts" do+ post_2.update!(user:)- expect(- described_class.new(user: user, date: date).post_query(user, date).map(&:id),- ).not_to include(post_2.id)+ expect(post_ids).to be_emptyend- it "excludes posts not created in the time range of the report" do+ it "excludes replies outside the year" dopost_2.update!(created_at: 2.years.ago)- expect(- described_class.new(user: user, date: date).post_query(user, date).map(&:id),- ).to be_empty+ expect(post_ids).to be_empty+ end++ it "excludes whispers when the user is a whisperer" do+ whisperers = Fabricate(:group)+ whisperers.add(user)+ SiteSetting.whispers_allowed_groups = whisperers.id.to_s+ post_2.update!(post_type: Post.types[:whisper])++ expect(post_ids).to be_emptyend- it "excludes posts by muted users" do- MutedUser.create!(user: user, muted_user: other_user)+ it "excludes hidden posts" do+ post_2.update!(hidden: true)- expect(- described_class.new(user: user, date: date).post_query(user, date).map(&:id),- ).to be_empty+ expect(post_ids).to be_emptyend- it "excludes posts by ignored users" do- IgnoredUser.create!(user: user, ignored_user: other_user, expiring_at: 1.year.from_now)+ it "excludes deleted posts" do+ post_2.trash!- expect(- described_class.new(user: user, date: date).post_query(user, date).map(&:id),- ).to be_empty+ expect(post_ids).to be_empty+ end++ it "excludes topics that are not publicly visible" do+ topic.update!(category: restricted_category)++ expect(post_ids).to be_emptyendend- describe ".like_query" do+ describe "#like_query" do+ subject(:like_ids) { described_class.new(user:, date:).like_query.map(&:id) }+fab!(:like_1) doFabricate(:user_action,action_type: UserAction::WAS_LIKED,acting_user: other_user,- user: post_1.user,+ user:,target_post: post_1,- target_topic: post_1.topic,+ target_topic: topic,created_at: random_datetime,)end@@ -110,99 +176,34 @@ RSpec.describe DiscourseRewind::Action::Fbff do:user_action,action_type: UserAction::WAS_LIKED,acting_user: user,- user: post_2.user,+ user: other_user,target_post: post_2,- target_topic: post_2.topic,+ target_topic: topic,created_at: random_datetime,)end- it "succesfully returns likes for the time period" do- expect(- described_class.new(user: user, date: date).like_query(user, date).map(&:id),- ).to match_array([like_1.id, like_2.id])+ it "returns the likes between users" do+ expect(like_ids).to contain_exactly(like_1.id, like_2.id)end- it "does not return likes outside the time period" do+ it "excludes likes outside the year" dolike_1.update!(created_at: 2.years.ago)- expect(- described_class.new(user: user, date: date).like_query(user, date).map(&:id),- ).not_to include(like_1.id)- end-- describe "user_id" do- it "excludes likes from deactivated users" do- other_user.deactivate(Discourse.system_user)-- expect(- described_class.new(user: user, date: date).like_query(user, date).map(&:id),- ).not_to include(like_1.id)- end-- it "excludes likes from suspended users" do- other_user.suspended_at = Time.now- other_user.suspended_till = 200.years.from_now- other_user.save!-- expect(- described_class.new(user: user, date: date).like_query(user, date).map(&:id),- ).not_to include(like_1.id)- end-- it "excludes likes from bot users" do- post_1.update!(user: bot)- like_1.update!(acting_user: bot)-- expect(- described_class.new(user: user, date: date).like_query(user, date).map(&:id),- ).not_to include(like_1.id)- end- end-- describe "acting_user_id" do- it "excludes likes from deactivated users" do- user.deactivate(Discourse.system_user)-- expect(- described_class.new(user: user, date: date).like_query(user, date).map(&:id),- ).not_to include(like_2.id)- end-- it "excludes likes from suspended users" do- user.suspended_at = Time.now- user.suspended_till = 200.years.from_now- user.save!-- expect(- described_class.new(user: user, date: date).like_query(user, date).map(&:id),- ).not_to include(like_2.id)- end-- it "excludes likes from bot users" do- post_2.update!(user: bot)- like_2.update!(acting_user: bot)-- expect(- described_class.new(user: user, date: date).like_query(user, date).map(&:id),- ).not_to include(like_2.id)- end+ expect(like_ids).to contain_exactly(like_2.id)end- it "excludes likes involving muted users" do- MutedUser.create!(user: user, muted_user: other_user)+ it "excludes likes on whispers and hidden posts" do+ post_1.update!(post_type: Post.types[:whisper])+ post_2.update!(hidden: true)- expect(- described_class.new(user: user, date: date).like_query(user, date).map(&:id),- ).to be_empty+ expect(like_ids).to be_emptyend- it "excludes likes involving ignored users" do- IgnoredUser.create!(user: user, ignored_user: other_user, expiring_at: 1.year.from_now)+ it "excludes topics that are not publicly visible" do+ topic.update!(category: restricted_category)- expect(- described_class.new(user: user, date: date).like_query(user, date).map(&:id),- ).to be_empty+ expect(like_ids).to be_emptyendendenddiff --git a/plugins/discourse-rewind/spec/actions/top_words_spec.rb b/plugins/discourse-rewind/spec/actions/top_words_spec.rbindex 1a5983e4706..318741adc0c 100644--- a/plugins/discourse-rewind/spec/actions/top_words_spec.rb+++ b/plugins/discourse-rewind/spec/actions/top_words_spec.rb@@ -85,6 +85,20 @@ RSpec.describe DiscourseRewind::Action::TopWords doendend+ it "only counts words from the user's publicly visible posts" do+ raw = "confidential " * 10+ restricted_topic =+ Fabricate(:topic, category: Fabricate(:private_category, group: Fabricate(:group)))+ [+ Fabricate(:private_message_post, user:, raw:, created_at: random_datetime),+ Fabricate(:post, user:, raw:, topic: restricted_topic, created_at: random_datetime),+ Fabricate(:post, user:, raw:, post_type: Post.types[:whisper], created_at: random_datetime),+ Fabricate(:post, user:, raw:, hidden: true, created_at: random_datetime),+ ].each { |post| SearchIndexer.index(post, force: true) }++ expect(call_report[:data].map { |word| word[:word] }).not_to include("confidential")+ end+context "with a large number of posts and words" dobefore do# Create posts with different frequencies of non-stop wordsdiff --git a/plugins/discourse-rewind/spec/plugin_helper.rb b/plugins/discourse-rewind/spec/plugin_helper.rbindex 5eed95bb838..aedce03685e 100644--- a/plugins/discourse-rewind/spec/plugin_helper.rb+++ b/plugins/discourse-rewind/spec/plugin_helper.rb@@ -1,9 +1,12 @@# frozen_string_literal: truemodule DiscourseRewindSpecHelper+ def date+ Time.zone.local(2021).all_year+ end+def call_report- # user + date should be defined via fab! in the spec- described_class.call(user:, date:, guardian: user.guardian)+ described_class.call(user:, date:)enddef random_datetime