The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
unstemmed words should still appear in search (#64560)
Oleksandr Solovyovf4c15ce5d7eb
diff --git a/src/metabase/search/appdb/specialization/postgres.clj b/src/metabase/search/appdb/specialization/postgres.cljindex 8aa5bef3f22..dbc76d04fb7 100644--- a/src/metabase/search/appdb/specialization/postgres.clj+++ b/src/metabase/search/appdb/specialization/postgres.clj@@ -54,14 +54,20 @@(defmethod specialization/extra-entry-fields :postgres [entity]{:search_vector- [:||- (search.util/weighted-tsvector "A" (or (:name entity) ""))- (search.util/weighted-tsvector "B" (or (:searchable_text entity) ""))]+ (cond-> [:||+ (search.util/weighted-tsvector "A" (or (:name entity) ""))+ (search.util/weighted-tsvector "B" (or (:searchable_text entity) ""))]+ ;; if stemming is not in a 'simple' mode, let's include non-stemmed words so that when your partially typed word+ ;; is not matching any stem you still have something to match; see tests for #60649+ (not= (search.util/tsv-language) "simple")+ (conj (search.util/weighted-tsvector "D" (or (:searchable_text entity) "") "simple"))):with_native_query_vector- [:||- (search.util/weighted-tsvector "A" (or (:name entity) ""))- (search.util/weighted-tsvector "B" (str/join " " (keep entity [:searchable_text :native_query])))]})+ (cond-> [:||+ (search.util/weighted-tsvector "A" (or (:name entity) ""))+ (search.util/weighted-tsvector "B" (str/join " " (keep entity [:searchable_text :native_query])))]+ (not= (search.util/tsv-language) "simple")+ (conj (search.util/weighted-tsvector "D" (or (:searchable_text entity) "") "simple")))});; See https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-RANKING;; 0 (the default) ignores the document length@@ -73,9 +79,15 @@;; 32 divides the rank by itself + 1(def ^:private ts-rank-normalization 0)+;; The order here is {D, C, B, A} - but see the link to the docs up there ^+;; A is the entity title+;; B is all searchable text+;; D is backup text with 'simple' stemmer, so it's not used in scoring, but mostly for completion+(def ^:private ts-rank-weights "{0, 0.2, 0.4, 1.0}")+(defmethod specialization/text-score :postgres[]- [:ts_rank :search_vector :query [:inline ts-rank-normalization]])+ [:ts_rank [:inline ts-rank-weights] :search_vector :query [:inline ts-rank-normalization]])(defmethod specialization/view-count-percentile-query :postgres[index-table p-value]diff --git a/src/metabase/search/util.clj b/src/metabase/search/util.cljindex 1d11ed93e9e..86bff5dd947 100644--- a/src/metabase/search/util.clj+++ b/src/metabase/search/util.clj@@ -153,6 +153,8 @@(defn weighted-tsvector"Create a weighted tsvector for Postgres full-text search with the given weight and text."- [weight text]- ;; tsvector has a max value size of 1048575 bytes, limit to less than that because the multiple values get concatenated together- [:setweight [:to_tsvector [:inline (tsv-language)] [:cast (u.str/limit-bytes text search.ingestion/max-searchable-value-length) :text]] [:inline weight]])+ ([weight text]+ (weighted-tsvector weight text (tsv-language)))+ ([weight text lang]+ ;; tsvector has a max value size of 1048575 bytes, limit to less than that because the multiple values get concatenated together+ [:setweight [:to_tsvector [:inline lang] [:cast (u.str/limit-bytes text search.ingestion/max-searchable-value-length) :text]] [:inline weight]]))diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.cljindex 98e2d048d87..4cab69f948b 100644--- a/test/metabase/search/appdb/index_test.clj+++ b/test/metabase/search/appdb/index_test.clj@@ -14,6 +14,7 @@[metabase.search.models.search-index-metadata :as search-index-metadata][metabase.search.spec :as search.spec][metabase.search.test-util :as search.tu]+ [metabase.search.util :as u.search][metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util :as u]@@ -57,6 +58,9 @@:model/Card {} {:name "Projected Revenue" :collection_id col-id#}:model/Card {} {:name "Employee Satisfaction" :collection_id col-id#}:model/Card {} {:name "Projected Satisfaction" :collection_id col-id#}+ :model/Card {} {:name "Organization Reference" :collection_id col-id#}+ ;; this is to show off that our storage can give false positives (stemming etc)+ :model/Card {} {:name "Nobody Expects Organi" :collection_id col-id#}:model/Database {db-id# :id} {:name "Indexed Database"}:model/Table {} {:name "Indexed Table", :db_id db-id#}](search.engine/reindex! :search.engine/appdb {:in-place? true})@@ -147,14 +151,22 @@(is (= 2 (index-hits "satisfaction -customer")))))))(deftest phrase-test- (with-index- (with-fulltext-filtering- ;; Less matches without an english dictionary- (is (= #_2 3 (index-hits "projected")))- (is (= 2 (index-hits "revenue")))- (is (= #_1 2 (index-hits "projected revenue")))- (testing "only sometimes do these occur sequentially in a phrase"- (is (= 1 (index-hits "\"projected revenue\"")))))))+ (mt/with-dynamic-fn-redefs [u.search/tsv-language (constantly "english")]+ (with-index+ (with-fulltext-filtering+ ;; Less matches without an english dictionary+ (is (= #_2 3 (index-hits "projected")))+ (is (= 2 (index-hits "revenue")))+ (is (= #_1 2 (index-hits "projected revenue")))+ (testing "Simple dictionary allows searching for unstemmed words #60649"+ ;; those two results are "Organization Reference" and "Nobody Expects Organi"+ (is (= 2 (index-hits "organ")))+ (is (= 2 (index-hits "organi"))))+ (testing "We get false-positive for 'Organi' here"+ ;; those two results are "Organization Reference" and "Nobody Expects Organi"+ (is (= 2 (index-hits "organization"))))+ (testing "only sometimes do these occur sequentially in a phrase"+ (is (= 1 (index-hits "\"projected revenue\""))))))))(defn ingest![model where-clause]diff --git a/test/metabase/search/appdb/scoring_test.clj b/test/metabase/search/appdb/scoring_test.cljindex 88d56236071..ea33c5885bb 100644--- a/test/metabase/search/appdb/scoring_test.clj+++ b/test/metabase/search/appdb/scoring_test.clj@@ -102,10 +102,10 @@;; Due to stemming, we do not distinguish between exact matches and those that differ slightly.(is (= [["card" 1 "orders"]["card" 4 "order"]- ;; We do not currently normalize the score based on the number of words in the vector / the coverage.+ ;; We do not currently normalize the score based on the number of words in the vector / the coverage.["card" 5 "orders, invoices, other stuff"]["card" 6 "ordering"]- ;; If the match is only in a secondary field, it is less preferred.+ ;; If the match is only in a secondary field, it is less preferred.["card" 3 "classified"]](search-results :text "order")))):h2