Skip to content

A platform for community discussion. Free, open, simple.

FIX: Only count public posts in Rewind Top Words and FBFF

Régis Hanolc2835fef529a
diff --git a/plugins/discourse-rewind/app/services/discourse_rewind/action/base_report.rb b/plugins/discourse-rewind/app/services/discourse_rewind/action/base_report.rb
index c796811d33e..193e3933e33 100644
--- a/plugins/discourse-rewind/app/services/discourse_rewind/action/base_report.rb
+++ b/plugins/discourse-rewind/app/services/discourse_rewind/action/base_report.rb
@@ -10,6 +10,14 @@ module DiscourseRewind
raise NotImplementedError
end
+ def self.publicly_visible_topics
+ Topic.listable_topics.visible.secured
+ end
+
+ def self.publicly_visible_posts
+ Post.visible.merge(publicly_visible_topics).where.not(post_type: Post.types[:whisper])
+ end
+
def self.enabled?
true
end
diff --git a/plugins/discourse-rewind/app/services/discourse_rewind/action/fbff.rb b/plugins/discourse-rewind/app/services/discourse_rewind/action/fbff.rb
index 77ebb4193ee..c23045f7153 100644
--- a/plugins/discourse-rewind/app/services/discourse_rewind/action/fbff.rb
+++ b/plugins/discourse-rewind/app/services/discourse_rewind/action/fbff.rb
@@ -1,7 +1,5 @@
# frozen_string_literal: true
-# Forum Best Friend Forever ranking
-# Score is informative only, do not show in UI
module DiscourseRewind
module Action
class Fbff < BaseReport
@@ -30,63 +28,23 @@ module DiscourseRewind
def call
return FakeData if should_use_fake_data?
- most_liked_users =
- like_query(user, date)
- .where(acting_user_id: user.id)
- .group(:user_id)
+ scores = Hash.new(0)
+ [
+ [like_query.where(acting_user_id: user.id), :user_id, LIKE_SCORE],
+ [like_query.where(user_id: user.id), :acting_user_id, LIKE_SCORE],
+ [post_query.where(posts: { user_id: user.id }), "replies.user_id", REPLY_SCORE],
+ [post_query.where(replies: { user_id: user.id }), "posts.user_id", REPLY_SCORE],
+ ].each do |query, friend_column, weight|
+ query
+ .where(friend_column => eligible_users)
+ .group(friend_column)
.order("COUNT(*) DESC")
.limit(MAX_SUMMARY_RESULTS)
- .pluck("user_actions.user_id, COUNT(*)")
- .map { |user_id, count| { user_id => count } }
- .reduce({}, :merge)
-
- most_liked_by_users =
- like_query(user, date)
- .where(user: user)
- .group(:acting_user_id)
- .order("COUNT(*) DESC")
- .limit(MAX_SUMMARY_RESULTS)
- .pluck("acting_user_id, COUNT(*)")
- .map { |acting_user_id, count| { acting_user_id => count } }
- .reduce({}, :merge)
-
- users_who_most_replied_me =
- post_query(user, date)
- .where(posts: { user_id: user.id })
- .group("replies.user_id")
- .order("COUNT(*) DESC")
- .limit(MAX_SUMMARY_RESULTS)
- .pluck("replies.user_id, COUNT(*)")
- .map { |user_id, count| { user_id => count } }
- .reduce({}, :merge)
-
- users_i_most_replied =
- post_query(user, date)
- .where("replies.user_id = ?", user.id)
- .group("posts.user_id")
- .order("COUNT(*) DESC")
- .limit(MAX_SUMMARY_RESULTS)
- .pluck("posts.user_id, COUNT(*)")
- .map { |user_id, count| { user_id => count } }
- .reduce({}, :merge)
-
- # NOTE: At some point maybe we want to include chat interactions
- # in the calculations here.
- fbffs = [
- apply_score(most_liked_users, LIKE_SCORE),
- apply_score(most_liked_by_users, LIKE_SCORE),
- apply_score(users_who_most_replied_me, REPLY_SCORE),
- apply_score(users_i_most_replied, REPLY_SCORE),
- ]
-
- fbff_id =
- fbffs
- .flatten
- .inject { |h1, h2| h1.merge(h2) { |_, v1, v2| v1 + v2 } }
- &.sort_by { |_, v| -v }
- &.first
- &.first
+ .count
+ .each { |user_id, count| scores[user_id] += count * weight }
+ end
+ fbff_id = scores.max_by(&:last)&.first
return if !fbff_id
{
@@ -98,66 +56,43 @@ module DiscourseRewind
}
end
- def post_query(user, date)
- Post
- .with(eligible_users: User.real.activated.not_suspended.select(:id))
- .joins(:topic)
- .includes(:topic)
- .where(
- "posts.post_type IN (?)",
- Topic.visible_post_types(user, include_moderator_actions: false),
- )
+ def post_query
+ self
+ .class
+ .publicly_visible_posts
.joins(
- "INNER JOIN posts replies ON posts.topic_id = replies.topic_id AND posts.reply_to_post_number = replies.post_number",
+ "INNER JOIN posts replies ON replies.topic_id = posts.topic_id AND replies.post_number = posts.reply_to_post_number",
)
- .joins(
- "INNER JOIN topics ON replies.topic_id = topics.id
- AND topics.archetype <> 'private_message'
- AND replies.post_type IN (#{Topic.visible_post_types(user, include_moderator_actions: false).join(",")})",
+ .where(posts: { post_type: Post.types[:regular], created_at: date })
+ .where(
+ replies: {
+ post_type: Post.types[:regular],
+ hidden: false,
+ created_at: date,
+ deleted_at: nil,
+ },
)
- .joins("INNER JOIN eligible_users eu ON eu.id = replies.user_id")
- .joins("INNER JOIN eligible_users eu2 ON eu2.id = posts.user_id")
- .where("replies.created_at BETWEEN ? AND ?", date.first, date.last)
- .where("posts.created_at BETWEEN ? AND ?", date.first, date.last)
.where("replies.user_id <> posts.user_id")
- .where(<<~SQL, user_id: user.id)
- NOT EXISTS (
- SELECT 1 FROM muted_users
- WHERE muted_users.user_id = :user_id
- AND muted_users.muted_user_id IN (replies.user_id, posts.user_id)
- )
- AND NOT EXISTS (
- SELECT 1 FROM ignored_users
- WHERE ignored_users.user_id = :user_id
- AND ignored_users.ignored_user_id IN (replies.user_id, posts.user_id)
- )
- SQL
end
- def like_query(user, date)
+ def like_query
UserAction
- .with(eligible_users: User.real.activated.not_suspended.select(:id))
.joins(:target_topic, :target_post)
- .joins("INNER JOIN eligible_users eu ON eu.id = user_actions.user_id")
- .joins("INNER JOIN eligible_users eu2 ON eu2.id = user_actions.acting_user_id")
- .where(created_at: date)
- .where(action_type: UserAction::WAS_LIKED)
- .where(<<~SQL, user_id: user.id)
- NOT EXISTS (
- SELECT 1 FROM muted_users
- WHERE muted_users.user_id = :user_id
- AND muted_users.muted_user_id IN (user_actions.user_id, user_actions.acting_user_id)
- )
- AND NOT EXISTS (
- SELECT 1 FROM ignored_users
- WHERE ignored_users.user_id = :user_id
- AND ignored_users.ignored_user_id IN (user_actions.user_id, user_actions.acting_user_id)
- )
- SQL
+ .merge(self.class.publicly_visible_topics)
+ .where(action_type: UserAction::WAS_LIKED, created_at: date)
+ .where(posts: { post_type: Post.types[:regular], hidden: false })
end
- def apply_score(users, score)
- users.map { |user_id, count| { user_id => count * score } }
+ private
+
+ def eligible_users
+ @eligible_users ||=
+ User
+ .real
+ .activated
+ .not_suspended
+ .where.not(id: user.muted_user_ids | user.ignored_user_ids)
+ .select(:id)
end
end
end
diff --git a/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rb b/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rb
index 97f42544666..fc8736e5c37 100644
--- a/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rb
+++ b/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rb
@@ -47,22 +47,16 @@ module DiscourseRewind
end
def word_query
- DB.query(<<~SQL, user_id: user.id, date_start: date.first, date_end: date.last)
+ posts = self.class.publicly_visible_posts.where(user_id: user.id, created_at: date)
+
+ DB.query(<<~SQL)
WITH popular_words AS (
SELECT
*
FROM
ts_stat(
$INNERSQL$
- SELECT
- search_data
- FROM
- post_search_data
- INNER JOIN
- posts ON posts.id = post_search_data.post_id
- WHERE
- posts.user_id = :user_id
- AND posts.created_at BETWEEN :date_start AND :date_end
+ #{posts.joins(:post_search_data).select(:search_data).to_sql}
$INNERSQL$
) AS search_data
WHERE LENGTH(word) >= 2
@@ -84,15 +78,7 @@ module DiscourseRewind
word as original_word
FROM
ts_stat ($INNERSQL$
- SELECT
- to_tsvector('simple',
- regexp_replace(raw, 'https?://[^\\s]+', ' ', 'g')
- )
- FROM
- posts AS p
- WHERE
- p.user_id = :user_id
- AND p.created_at BETWEEN :date_start AND :date_end
+ #{posts.select("to_tsvector('simple', regexp_replace(raw, 'https?://[^\\s]+', ' ', 'g'))").to_sql}
$INNERSQL$)
WHERE LENGTH(word) >= 2
AND word ~ '^[a-zA-Z]+$'
diff --git a/plugins/discourse-rewind/lib/discourse_rewind/fetch_reports_helper.rb b/plugins/discourse-rewind/lib/discourse_rewind/fetch_reports_helper.rb
index 84fb3ee2d92..2d401c13857 100644
--- a/plugins/discourse-rewind/lib/discourse_rewind/fetch_reports_helper.rb
+++ b/plugins/discourse-rewind/lib/discourse_rewind/fetch_reports_helper.rb
@@ -4,7 +4,7 @@ module DiscourseRewind
module FetchReportsHelper
extend self
- REWIND_REPORT_VERSION = "2"
+ REWIND_REPORT_VERSION = "3"
CACHE_DURATION = Rails.env.development? ? 10.seconds : 3.days
def cache_key(user_id, year)
diff --git a/plugins/discourse-rewind/spec/actions/base_report_spec.rb b/plugins/discourse-rewind/spec/actions/base_report_spec.rb
new file mode 100644
index 00000000000..8bd4386e73f
--- /dev/null
+++ b/plugins/discourse-rewind/spec/actions/base_report_spec.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseRewind::Action::BaseReport do
+ fab!(:public_topic, :topic)
+ fab!(:unlisted_topic) { Fabricate(:topic, visible: false) }
+ fab!(:private_message_topic)
+ fab!(:restricted_topic) do
+ Fabricate(:topic, category: Fabricate(:private_category, group: Fabricate(:group)))
+ end
+ fab!(:topics) { [public_topic, unlisted_topic, private_message_topic, restricted_topic] }
+
+ describe ".publicly_visible_topics" do
+ it "only returns listed topics in public categories" do
+ expect(described_class.publicly_visible_topics.where(id: topics)).to contain_exactly(
+ public_topic,
+ )
+ end
+ end
+
+ describe ".publicly_visible_posts" do
+ it "only returns visible, non-whisper posts in publicly visible topics" do
+ public_post = Fabricate(:post, topic: public_topic)
+ posts = [
+ public_post,
+ Fabricate(:post, topic: public_topic, hidden: true),
+ Fabricate(:post, topic: public_topic, post_type: Post.types[:whisper]),
+ *topics.drop(1).map { |topic| Fabricate(:post, topic:) },
+ ]
+
+ expect(described_class.publicly_visible_posts.where(id: posts)).to contain_exactly(
+ public_post,
+ )
+ end
+ end
+end
diff --git a/plugins/discourse-rewind/spec/actions/fbff_spec.rb b/plugins/discourse-rewind/spec/actions/fbff_spec.rb
index 2259528bc07..8f837ce5b42 100644
--- a/plugins/discourse-rewind/spec/actions/fbff_spec.rb
+++ b/plugins/discourse-rewind/spec/actions/fbff_spec.rb
@@ -1,107 +1,173 @@
# frozen_string_literal: true
RSpec.describe DiscourseRewind::Action::Fbff do
- fab!(:date) { Date.new(2021).all_year }
fab!(:user)
fab!(:other_user, :user)
- fab!(:bot)
-
fab!(:topic)
- fab!(:post_1) { Fabricate(:post, topic: topic, user: user, created_at: random_datetime) }
+ fab!(:post_1) { Fabricate(:post, topic:, user:, created_at: random_datetime) }
fab!(:post_2) do
Fabricate(
:post,
- topic: topic,
+ topic:,
user: other_user,
created_at: random_datetime,
- reply_to_post_number: 1,
+ reply_to_post_number: post_1.post_number,
)
end
fab!(:post_3) do
Fabricate(
:post,
- topic: topic,
- user: user,
+ topic:,
+ user:,
created_at: random_datetime,
reply_to_post_number: post_2.post_number,
)
end
+ fab!(:restricted_category) { Fabricate(:private_category, group: Fabricate(:group)) }
+
+ describe ".call" do
+ it "ranks a reply higher than a like" do
+ liker = Fabricate(:user)
+ 9.times do
+ liked_post = Fabricate(:post, user:, created_at: random_datetime)
+ Fabricate(
+ :user_action,
+ action_type: UserAction::WAS_LIKED,
+ acting_user: liker,
+ user:,
+ target_post: liked_post,
+ target_topic: liked_post.topic,
+ created_at: random_datetime,
+ )
+ end
- describe ".post_query" do
- it "includes the correct posts for the time period based on replies between users" do
- expect(
- described_class.new(user: user, date: date).post_query(user, date).map(&:id),
- ).to match_array([post_2.id, post_3.id])
+ expect(call_report[:data][:fbff][:id]).to eq(other_user.id)
end
- it "excludes posts by bot users" do
- post_2.update!(user: bot)
+ context "when there are only likes" do
+ fab!(:friend, :user)
- expect(
- described_class.new(user: user, date: date).post_query(user, date).map(&:id),
- ).not_to include(post_2.id)
+ before { [post_2, post_3].each(&:trash!) }
+
+ def like(acting_user:, liked_user:)
+ liked_post = Fabricate(:post, user: liked_user, created_at: random_datetime)
+ Fabricate(
+ :user_action,
+ action_type: UserAction::WAS_LIKED,
+ acting_user:,
+ user: liked_user,
+ target_post: liked_post,
+ target_topic: liked_post.topic,
+ created_at: random_datetime,
+ )
+ end
+
+ it "counts likes the user gave" do
+ like(acting_user: user, liked_user: friend)
+
+ expect(call_report[:data][:fbff][:id]).to eq(friend.id)
+ end
+
+ it "counts likes the user received" do
+ like(acting_user: friend, liked_user: user)
+
+ expect(call_report[:data][:fbff][:id]).to eq(friend.id)
+ end
end
- it "excludes posts by deactivated users" do
- user.deactivate(Discourse.system_user)
+ context "when the only friend exchanges likes and replies with the user" do
+ before do
+ [[other_user, user, post_1], [user, other_user, post_2]].each do |acting_user, liked, post|
+ Fabricate(
+ :user_action,
+ action_type: UserAction::WAS_LIKED,
+ acting_user:,
+ user: liked,
+ target_post: post,
+ target_topic: topic,
+ created_at: random_datetime,
+ )
+ end
+ end
+
+ it "ignores them when they are not active" do
+ other_user.deactivate(Discourse.system_user)
- expect(
- described_class.new(user: user, date: date).post_query(user, date).map(&:id),
- ).not_to include(post_2.id)
+ expect(call_report).to be_nil
+ end
+
+ it "ignores them when they are muted" do
+ Fabricate(:muted_user, user:, muted_user: other_user)
+
+ expect(call_report).to be_nil
+ end
+
+ it "ignores them when they are ignored" do
+ Fabricate(:ignored_user, user:, ignored_user: other_user)
+
+ expect(call_report).to be_nil
+ end
end
+ end
- it "excludes posts by suspended users" do
- user.suspended_at = Time.now
- user.suspended_till = 200.years.from_now
- user.save!
+ describe "#post_query" do
+ subject(:post_ids) { described_class.new(user:, date:).post_query.map(&:id) }
- expect(
- described_class.new(user: user, date: date).post_query(user, date).map(&:id),
- ).not_to include(post_2.id)
+ it "returns the replies between users" do
+ expect(post_ids).to contain_exactly(post_2.id, post_3.id)
end
- it "excludes moderator action posts" do
- post_2.update!(post_type: Post.types[:moderator_action])
+ it "excludes replies to the user's own posts" do
+ post_2.update!(user:)
- expect(
- described_class.new(user: user, date: date).post_query(user, date).map(&:id),
- ).not_to include(post_2.id)
+ expect(post_ids).to be_empty
end
- it "excludes posts not created in the time range of the report" do
+ it "excludes replies outside the year" do
post_2.update!(created_at: 2.years.ago)
- expect(
- described_class.new(user: user, date: date).post_query(user, date).map(&:id),
- ).to be_empty
+ expect(post_ids).to be_empty
+ end
+
+ it "excludes whispers when the user is a whisperer" do
+ whisperers = Fabricate(:group)
+ whisperers.add(user)
+ SiteSetting.whispers_allowed_groups = whisperers.id.to_s
+ post_2.update!(post_type: Post.types[:whisper])
+
+ expect(post_ids).to be_empty
end
- it "excludes posts by muted users" do
- MutedUser.create!(user: user, muted_user: other_user)
+ it "excludes hidden posts" do
+ post_2.update!(hidden: true)
- expect(
- described_class.new(user: user, date: date).post_query(user, date).map(&:id),
- ).to be_empty
+ expect(post_ids).to be_empty
end
- it "excludes posts by ignored users" do
- IgnoredUser.create!(user: user, ignored_user: other_user, expiring_at: 1.year.from_now)
+ it "excludes deleted posts" do
+ post_2.trash!
- expect(
- described_class.new(user: user, date: date).post_query(user, date).map(&:id),
- ).to be_empty
+ expect(post_ids).to be_empty
+ end
+
+ it "excludes topics that are not publicly visible" do
+ topic.update!(category: restricted_category)
+
+ expect(post_ids).to be_empty
end
end
- describe ".like_query" do
+ describe "#like_query" do
+ subject(:like_ids) { described_class.new(user:, date:).like_query.map(&:id) }
+
fab!(:like_1) do
Fabricate(
:user_action,
action_type: UserAction::WAS_LIKED,
acting_user: other_user,
- user: post_1.user,
+ user:,
target_post: post_1,
- target_topic: post_1.topic,
+ target_topic: topic,
created_at: random_datetime,
)
end
@@ -110,99 +176,34 @@ RSpec.describe DiscourseRewind::Action::Fbff do
:user_action,
action_type: UserAction::WAS_LIKED,
acting_user: user,
- user: post_2.user,
+ user: other_user,
target_post: post_2,
- target_topic: post_2.topic,
+ target_topic: topic,
created_at: random_datetime,
)
end
- it "succesfully returns likes for the time period" do
- expect(
- described_class.new(user: user, date: date).like_query(user, date).map(&:id),
- ).to match_array([like_1.id, like_2.id])
+ it "returns the likes between users" do
+ expect(like_ids).to contain_exactly(like_1.id, like_2.id)
end
- it "does not return likes outside the time period" do
+ it "excludes likes outside the year" do
like_1.update!(created_at: 2.years.ago)
- expect(
- described_class.new(user: user, date: date).like_query(user, date).map(&:id),
- ).not_to include(like_1.id)
- end
-
- describe "user_id" do
- it "excludes likes from deactivated users" do
- other_user.deactivate(Discourse.system_user)
-
- expect(
- described_class.new(user: user, date: date).like_query(user, date).map(&:id),
- ).not_to include(like_1.id)
- end
-
- it "excludes likes from suspended users" do
- other_user.suspended_at = Time.now
- other_user.suspended_till = 200.years.from_now
- other_user.save!
-
- expect(
- described_class.new(user: user, date: date).like_query(user, date).map(&:id),
- ).not_to include(like_1.id)
- end
-
- it "excludes likes from bot users" do
- post_1.update!(user: bot)
- like_1.update!(acting_user: bot)
-
- expect(
- described_class.new(user: user, date: date).like_query(user, date).map(&:id),
- ).not_to include(like_1.id)
- end
- end
-
- describe "acting_user_id" do
- it "excludes likes from deactivated users" do
- user.deactivate(Discourse.system_user)
-
- expect(
- described_class.new(user: user, date: date).like_query(user, date).map(&:id),
- ).not_to include(like_2.id)
- end
-
- it "excludes likes from suspended users" do
- user.suspended_at = Time.now
- user.suspended_till = 200.years.from_now
- user.save!
-
- expect(
- described_class.new(user: user, date: date).like_query(user, date).map(&:id),
- ).not_to include(like_2.id)
- end
-
- it "excludes likes from bot users" do
- post_2.update!(user: bot)
- like_2.update!(acting_user: bot)
-
- expect(
- described_class.new(user: user, date: date).like_query(user, date).map(&:id),
- ).not_to include(like_2.id)
- end
+ expect(like_ids).to contain_exactly(like_2.id)
end
- it "excludes likes involving muted users" do
- MutedUser.create!(user: user, muted_user: other_user)
+ it "excludes likes on whispers and hidden posts" do
+ post_1.update!(post_type: Post.types[:whisper])
+ post_2.update!(hidden: true)
- expect(
- described_class.new(user: user, date: date).like_query(user, date).map(&:id),
- ).to be_empty
+ expect(like_ids).to be_empty
end
- it "excludes likes involving ignored users" do
- IgnoredUser.create!(user: user, ignored_user: other_user, expiring_at: 1.year.from_now)
+ it "excludes topics that are not publicly visible" do
+ topic.update!(category: restricted_category)
- expect(
- described_class.new(user: user, date: date).like_query(user, date).map(&:id),
- ).to be_empty
+ expect(like_ids).to be_empty
end
end
end
diff --git a/plugins/discourse-rewind/spec/actions/top_words_spec.rb b/plugins/discourse-rewind/spec/actions/top_words_spec.rb
index 1a5983e4706..318741adc0c 100644
--- a/plugins/discourse-rewind/spec/actions/top_words_spec.rb
+++ b/plugins/discourse-rewind/spec/actions/top_words_spec.rb
@@ -85,6 +85,20 @@ RSpec.describe DiscourseRewind::Action::TopWords do
end
end
+ it "only counts words from the user's publicly visible posts" do
+ raw = "confidential " * 10
+ restricted_topic =
+ Fabricate(:topic, category: Fabricate(:private_category, group: Fabricate(:group)))
+ [
+ Fabricate(:private_message_post, user:, raw:, created_at: random_datetime),
+ Fabricate(:post, user:, raw:, topic: restricted_topic, created_at: random_datetime),
+ Fabricate(:post, user:, raw:, post_type: Post.types[:whisper], created_at: random_datetime),
+ Fabricate(:post, user:, raw:, hidden: true, created_at: random_datetime),
+ ].each { |post| SearchIndexer.index(post, force: true) }
+
+ expect(call_report[:data].map { |word| word[:word] }).not_to include("confidential")
+ end
+
context "with a large number of posts and words" do
before do
# Create posts with different frequencies of non-stop words
diff --git a/plugins/discourse-rewind/spec/plugin_helper.rb b/plugins/discourse-rewind/spec/plugin_helper.rb
index 5eed95bb838..aedce03685e 100644
--- a/plugins/discourse-rewind/spec/plugin_helper.rb
+++ b/plugins/discourse-rewind/spec/plugin_helper.rb
@@ -1,9 +1,12 @@
# frozen_string_literal: true
module DiscourseRewindSpecHelper
+ def date
+ Time.zone.local(2021).all_year
+ end
+
def call_report
- # user + date should be defined via fab! in the spec
- described_class.call(user:, date:, guardian: user.guardian)
+ described_class.call(user:, date:)
end
def random_datetime