A platform for community discussion. Free, open, simple.
FIX: Use the site locale for Rewind Top Words
Régis Hanol6ae495a4d4e7
diff --git a/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rb b/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rbindex fc8736e5c37..b8a2d70c8c3 100644--- a/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rb+++ b/plugins/discourse-rewind/app/services/discourse_rewind/action/top_words.rb@@ -3,14 +3,33 @@module DiscourseRewindmodule Actionclass TopWords < BaseReport- STEMMER_WORKAROUNDS = {- "discour" => "discourse",- "discours" => "discourse",- "topical" => "topic",- "topically" => "topic",- "categori" => "category",- "categor" => "category",- }.freeze+ IGNORED_WORDS = %w[+ com+ org+ net+ io+ dev+ co+ uk+ http+ https+ www+ github+ gitlab+ youtube+ slack+ discord+ drive+ ].freeze+ SIMPLE_TS_CONFIG = "simple"+ WORD_PATTERN = "^[^[:digit:][:punct:][:space:]]{2,}$"+ WORD_COUNT = 5+ CANDIDATE_COUNT = 100+ LINKS_REGEX = "https?://[^\\s]+"+ RAW_TSVECTOR_SQL =+ "to_tsvector('#{SIMPLE_TS_CONFIG}', regexp_replace(raw, '#{LINKS_REGEX}', ' ', 'g'))"FakeData = {data: [@@ -26,83 +45,56 @@ module DiscourseRewinddef callreturn FakeData if should_use_fake_data?- words = word_query+ data = word_query.map { |row| { word: row.original_word, score: row.ndoc + row.nentry } }- word_score =- words- .map do |word_data|- # Little cheat, since sometimes the stemming process uses words- # that we wouldn't normally use, especially for discourse-specific- # terms like "discourse" and "topic".- if STEMMER_WORKAROUNDS.key?(word_data.original_word)- word_data.original_word = STEMMER_WORKAROUNDS[word_data.original_word]- end-- { word: word_data.original_word, score: word_data.ndoc + word_data.nentry }- end- .sort_by! { |w| -w[:score] }- .take(5)-- { data: word_score, identifier: "top-words" }+ { data:, identifier: "top-words" }end+ private+def word_queryposts = self.class.publicly_visible_posts.where(user_id: user.id, created_at: date)+ stem = "strip(to_tsvector('#{Search.ts_config}', #{Search.wrap_unaccent("word")}))"DB.query(<<~SQL)WITH popular_words AS (SELECT- *+ word, ndoc, nentryFROM- ts_stat(- $INNERSQL$- #{posts.joins(:post_search_data).select(:search_data).to_sql}- $INNERSQL$- ) AS search_data- WHERE LENGTH(word) >= 2- AND word ~ '^[a-zA-Z]+$'- AND word NOT IN (- 'com', 'org', 'net', 'io', 'dev', 'co', 'uk', 'http', 'https',- 'www', 'github', 'gitlab', 'google', 'youtube', 'twitter',- 'slack', 'discord', 'drive'- )+ ts_stat($INNERSQL$+ #{posts.joins(:post_search_data).select(:search_data).to_sql}+ $INNERSQL$)+ WHERE+ word ~ '#{WORD_PATTERN}'+ AND word NOT IN (+ SELECT unnest(tsvector_to_array(strip(to_tsvector('#{Search.ts_config}', ignored))))+ FROM unnest(ARRAY[#{IGNORED_WORDS.map { |word| "'#{word}'" }.join(", ")}]) AS ignored+ )ORDER BY- nentry DESC,- ndoc DESC,- word- LIMIT- 100+ nentry DESC, ndoc DESC, word+ LIMIT #{CANDIDATE_COUNT}), lex AS (- SELECT- DISTINCT ON (lexeme) to_tsvector('english', word) as lexeme,- word as original_word+ SELECT DISTINCT ON (stem)+ #{stem} AS stem,+ word AS original_wordFROM- ts_stat ($INNERSQL$- #{posts.select("to_tsvector('simple', regexp_replace(raw, 'https?://[^\\s]+', ' ', 'g'))").to_sql}+ ts_stat($INNERSQL$+ #{posts.select(RAW_TSVECTOR_SQL).to_sql}$INNERSQL$)- WHERE LENGTH(word) >= 2- AND word ~ '^[a-zA-Z]+$'- ), ranked_words AS (- SELECT- popular_words.*, lex.original_word,- ROW_NUMBER() OVER (PARTITION BY word ORDER BY LENGTH(original_word) DESC) AS rn- FROM- popular_words- INNER JOIN- lex ON lex.lexeme @@ to_tsquery('english', popular_words.word)+ WHERE+ word ~ '#{WORD_PATTERN}'+ ORDER BY+ stem, nentry DESC, word)SELECT- word,- ndoc,- nentry,- original_word+ ndoc, nentry, original_wordFROM- ranked_words- WHERE- rn = 1+ popular_words+ INNER JOIN+ lex ON lex.stem = strip(to_tsvector('#{SIMPLE_TS_CONFIG}', popular_words.word))ORDER BY- ndoc + nentry DESC- LIMIT 10+ ndoc + nentry DESC, popular_words.word+ LIMIT #{WORD_COUNT}SQLendenddiff --git a/plugins/discourse-rewind/spec/actions/top_words_spec.rb b/plugins/discourse-rewind/spec/actions/top_words_spec.rbindex 318741adc0c..493b10c1ce9 100644--- a/plugins/discourse-rewind/spec/actions/top_words_spec.rb+++ b/plugins/discourse-rewind/spec/actions/top_words_spec.rb@@ -1,177 +1,79 @@# frozen_string_literal: trueRSpec.describe DiscourseRewind::Action::TopWords do- fab!(:date) { Date.new(2021).all_year }- fab!(:user)- fab!(:other_user, :user)-- fab!(:post1) do- Fabricate(- :post,- user: user,- raw: "apple orange banana apple apple orange",- created_at: random_datetime,- )- end- fab!(:post2) do- Fabricate(:post, user: user, raw: "cucumber tomato banana orange", created_at: random_datetime)- end- fab!(:post3) do- Fabricate(:post, user: user, raw: "grape watermelon mango", created_at: random_datetime)- end- fab!(:post4) do- Fabricate(:post, user: user, raw: "apple banana grape apple", created_at: random_datetime)- end- fab!(:post5) do- Fabricate(:post, user: user, raw: "apple orange apple apple", created_at: random_datetime)- end- fab!(:other_user_post) do- Fabricate(:post, user: other_user, raw: "apple apple apple", created_at: random_datetime)+ def index(post)+ SearchIndexer.index(post, force: true)end- before do+ fab!(:user)+ fab!(:posts) doSearchIndexer.enable- [post1, post2, post3, post4, post5, other_user_post].each do |post|- SearchIndexer.index(post, force: true)+ [+ "apple orange banana apple apple orange",+ "cucumber tomato banana orange",+ "grape watermelon mango",+ "apple banana grape apple",+ "apple orange apple apple",+ ].map do |raw|+ Fabricate(:post, user:, raw:, created_at: random_datetime).tap { |post| index(post) }endend- describe ".call" do- it "limits top words to 5" do- result = call_report+ before { SearchIndexer.enable }- expect(result[:data].length).to eq(5)+ describe ".call" do+ it "returns the five most used words with their score" do+ expect(call_report).to eq(+ data: [+ { word: "apple", score: 11 },+ { word: "orange", score: 7 },+ { word: "banana", score: 6 },+ { word: "grape", score: 4 },+ { word: "cucumber", score: 2 },+ ],+ identifier: "top-words",+ )end- it "returns top words ordered by frequency" do- result = call_report-- expect(result[:identifier]).to eq("top-words")-- words = result[:data]-- expect(words.first[:word]).to eq("apple")- expect(words.second[:word]).to eq("orange")- expect(words.third[:word]).to eq("banana")+ it "shows the most used form of a word" do+ post = Fabricate(:post, user:, raw: "#{"releases " * 9}release", created_at: random_datetime)+ index(post)- expect(words.map { |w| w[:word] }).to include("apple", "orange", "banana", "grape")- expect(words.map { |w| w[:score] }).to eq(words.map { |w| w[:score] }.sort.reverse)+ expect(call_report[:data].map { |word| word[:word] }).to include("releases")end- context "when a post is deleted" do- before do- post1.trash!(Discourse.system_user)- post1.post_search_data.destroy!- end+ it "ignores link and domain words" do+ post = Fabricate(:post, user:, raw: "github github github", created_at: random_datetime)+ index(post)- it "does not include words from deleted posts" do- result = call_report-- words = result[:data]-- apple = words.find { |w| w[:word] == "apple" }- expect(apple[:score]).to be < 9- end+ expect(call_report[:data].map { |word| word[:word] }).not_to include("github")end- context "when posts are from another user" do- it "does not include words from other users' posts" do- result = call_report+ it "uses the stemmer and accent handling of the site locale" do+ SiteSetting.default_locale = "fr"+ SiteSetting.search_ignore_accents = true+ raw = ("chevaux " * 20) + ("élèves " * 20)+ post = Fabricate(:post, user:, raw:, created_at: random_datetime)+ index(post)- words = result[:data]- apple_score = words.find { |w| w[:word] == "apple" }[:score]-- expect(apple_score).to be < 12- end+ expect(call_report[:data].map { |word| word[:word] }).to include("chevaux", "élèves")endit "only counts words from the user's publicly visible posts" doraw = "confidential " * 10- restricted_topic =- Fabricate(:topic, category: Fabricate(:private_category, group: Fabricate(:group)))[+ Fabricate(:post, user:, raw: "pumpkin " * 10, created_at: random_datetime),+ Fabricate(:post, raw:, created_at: random_datetime),Fabricate(:private_message_post, user:, raw:, created_at: random_datetime),- Fabricate(:post, user:, raw:, topic: restricted_topic, created_at: random_datetime),- Fabricate(:post, user:, raw:, post_type: Post.types[:whisper], created_at: random_datetime),- Fabricate(:post, user:, raw:, hidden: true, created_at: random_datetime),- ].each { |post| SearchIndexer.index(post, force: true) }-- expect(call_report[:data].map { |word| word[:word] }).not_to include("confidential")- end-- context "with a large number of posts and words" do- before do- # Create posts with different frequencies of non-stop words- 10.times do |i|- post =- Fabricate(- :post,- user: user,- raw: "#{frequent_word} #{frequent_word} #{frequent_word} #{infrequent_word}",- created_at: random_datetime,- )- SearchIndexer.index(post, force: true)- end- end-- let(:frequent_word) { "zucchini" }- let(:infrequent_word) { "xylophone" }-- it "ranks high frequency words higher than low frequency words" do- result = call_report-- words = result[:data]- frequent_word_entry = words.find { |w| w[:word] == frequent_word }- infrequent_word_entry = words.find { |w| w[:word] == infrequent_word }-- expect(frequent_word_entry).to be_present- expect(infrequent_word_entry).to be_present-- expect(frequent_word_entry[:score]).to be > infrequent_word_entry[:score]- end- end-- context "with stemmer workaround words" do- before do- DiscourseRewind::Action::TopWords- .any_instance- .stubs(:word_query)- .returns(- [- OpenStruct.new(original_word: "discours", ndoc: 5, nentry: 10),- OpenStruct.new(original_word: "topical", ndoc: 3, nentry: 7),- OpenStruct.new(original_word: "categori", ndoc: 4, nentry: 6),- ],- )- end-- it "applies STEMMER_WORKAROUNDS to replace stemmed words with preferred forms" do- result = call_report-- words = result[:data].map { |w| w[:word] }-- expect(words).to include("discourse")- expect(words).to include("topic")- expect(words).to include("category")- expect(words).not_to include("topical")- expect(words).not_to include("categori")- expect(words).not_to include("discours")- end- end- end-- context "when in rails development mode" do- before { Rails.env.stubs(:development?).returns(true) }-- it "returns fake data" do- result = call_report-- expect(result[:identifier]).to eq("top-words")- expect(result[:data].length).to eq(5)- expect(result[:data].first[:word]).to eq("seven")- expect(result[:data].first[:score]).to eq(100)- expect(result[:data].second[:word]).to eq("longest")- expect(result[:data].second[:score]).to eq(90)+ ].each { |post| index(post) }++ expect(call_report[:data].map { |word| word[:word] }).to contain_exactly(+ "apple",+ "orange",+ "banana",+ "grape",+ "pumpkin",+ )endendend