Skip to content

A platform for community discussion. Free, open, simple.

FIX: Show Rewind Top Words for Chinese and Japanese sites

Régis Hanol433cfa5f016a
diff --git a/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rb b/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rb
index 1d031cbd088..d5f2379f3dd 100644
--- a/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rb
+++ b/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rb
@@ -63,9 +63,15 @@ module DiscourseRewind
SQL
end
+ def segmented_search_data?
+ Search.ts_config == SIMPLE_TS_CONFIG &&
+ (Search.segment_chinese? || Search.segment_japanese?)
+ end
+
def word_query
posts = self.class.publicly_visible_posts.where(user_id: user.id, created_at: date)
stem = "strip(to_tsvector('#{Search.ts_config}', #{Search.wrap_unaccent("word")}))"
+ lex_join = segmented_search_data? ? "LEFT JOIN" : "INNER JOIN"
DB.query(<<~SQL)
WITH popular_words AS (
@@ -98,10 +104,10 @@ module DiscourseRewind
stem, nentry DESC, word
)
SELECT
- ndoc, nentry, original_word
+ ndoc, nentry, COALESCE(original_word, popular_words.word) AS original_word
FROM
popular_words
- INNER JOIN
+ #{lex_join}
lex ON lex.stem = strip(to_tsvector('#{SIMPLE_TS_CONFIG}', popular_words.word))
ORDER BY
ndoc + nentry DESC, popular_words.word
diff --git a/plugins/discourse-rewind/spec/actions/top_words_spec.rb b/plugins/discourse-rewind/spec/actions/top_words_spec.rb
index 97f3282a4c3..83ccd27eda9 100644
--- a/plugins/discourse-rewind/spec/actions/top_words_spec.rb
+++ b/plugins/discourse-rewind/spec/actions/top_words_spec.rb
@@ -100,6 +100,31 @@ RSpec.describe DiscourseRewind::Action::TopWords do
expect(call_report[:data].map { |word| word[:word] }).to include("chevaux", "élèves")
end
+ it "shows words from locales whose search data is segmented" do
+ SiteSetting.default_locale = "ja"
+ post = Fabricate(:post, user:, raw: "東京に行きました。" * 20, created_at: random_datetime)
+ index(post)
+
+ expect(call_report[:data].map { |word| word[:word] }).to include("まし")
+ end
+
+ it "ignores unstemmed words on English sites that segment Japanese" do
+ SiteSetting.search_tokenize_japanese = true
+
+ expect(call_report[:data].map { |word| word[:word] }).to eq(
+ %w[apple orange banana grape cucumber],
+ )
+ end
+
+ it "ignores quoted words on sites whose search data is not segmented" do
+ SiteSetting.default_locale = "ko"
+ raw = "[quote=\"someone, post:1, topic:2\"]\n#{"인용 " * 20}\n[/quote]\n#{"나의 " * 5}"
+ post = Fabricate(:post, user:, raw:, created_at: random_datetime)
+ index(post)
+
+ expect(call_report[:data].map { |word| word[:word] }).to include("나의").and exclude("인용")
+ end
+
it "only counts words from the user's publicly visible posts" do
raw = "confidential " * 10
[