Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

unstemmed words should still appear in search (#64560)

Oleksandr Solovyovf4c15ce5d7eb
diff --git a/src/metabase/search/appdb/specialization/postgres.clj b/src/metabase/search/appdb/specialization/postgres.clj
index 8aa5bef3f22..dbc76d04fb7 100644
--- a/src/metabase/search/appdb/specialization/postgres.clj
+++ b/src/metabase/search/appdb/specialization/postgres.clj
@@ -54,14 +54,20 @@
(defmethod specialization/extra-entry-fields :postgres [entity]
{:search_vector
- [:||
- (search.util/weighted-tsvector "A" (or (:name entity) ""))
- (search.util/weighted-tsvector "B" (or (:searchable_text entity) ""))]
+ (cond-> [:||
+ (search.util/weighted-tsvector "A" (or (:name entity) ""))
+ (search.util/weighted-tsvector "B" (or (:searchable_text entity) ""))]
+ ;; if stemming is not in a 'simple' mode, let's include non-stemmed words so that when your partially typed word
+ ;; is not matching any stem you still have something to match; see tests for #60649
+ (not= (search.util/tsv-language) "simple")
+ (conj (search.util/weighted-tsvector "D" (or (:searchable_text entity) "") "simple")))
:with_native_query_vector
- [:||
- (search.util/weighted-tsvector "A" (or (:name entity) ""))
- (search.util/weighted-tsvector "B" (str/join " " (keep entity [:searchable_text :native_query])))]})
+ (cond-> [:||
+ (search.util/weighted-tsvector "A" (or (:name entity) ""))
+ (search.util/weighted-tsvector "B" (str/join " " (keep entity [:searchable_text :native_query])))]
+ (not= (search.util/tsv-language) "simple")
+ (conj (search.util/weighted-tsvector "D" (or (:searchable_text entity) "") "simple")))})
;; See https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-RANKING
;; 0 (the default) ignores the document length
@@ -73,9 +79,15 @@
;; 32 divides the rank by itself + 1
(def ^:private ts-rank-normalization 0)
+;; The order here is {D, C, B, A} - but see the link to the docs up there ^
+;; A is the entity title
+;; B is all searchable text
+;; D is backup text with 'simple' stemmer, so it's not used in scoring, but mostly for completion
+(def ^:private ts-rank-weights "{0, 0.2, 0.4, 1.0}")
+
(defmethod specialization/text-score :postgres
[]
- [:ts_rank :search_vector :query [:inline ts-rank-normalization]])
+ [:ts_rank [:inline ts-rank-weights] :search_vector :query [:inline ts-rank-normalization]])
(defmethod specialization/view-count-percentile-query :postgres
[index-table p-value]
diff --git a/src/metabase/search/util.clj b/src/metabase/search/util.clj
index 1d11ed93e9e..86bff5dd947 100644
--- a/src/metabase/search/util.clj
+++ b/src/metabase/search/util.clj
@@ -153,6 +153,8 @@
(defn weighted-tsvector
"Create a weighted tsvector for Postgres full-text search with the given weight and text."
- [weight text]
- ;; tsvector has a max value size of 1048575 bytes, limit to less than that because the multiple values get concatenated together
- [:setweight [:to_tsvector [:inline (tsv-language)] [:cast (u.str/limit-bytes text search.ingestion/max-searchable-value-length) :text]] [:inline weight]])
+ ([weight text]
+ (weighted-tsvector weight text (tsv-language)))
+ ([weight text lang]
+ ;; tsvector has a max value size of 1048575 bytes, limit to less than that because the multiple values get concatenated together
+ [:setweight [:to_tsvector [:inline lang] [:cast (u.str/limit-bytes text search.ingestion/max-searchable-value-length) :text]] [:inline weight]]))
diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.clj
index 98e2d048d87..4cab69f948b 100644
--- a/test/metabase/search/appdb/index_test.clj
+++ b/test/metabase/search/appdb/index_test.clj
@@ -14,6 +14,7 @@
[metabase.search.models.search-index-metadata :as search-index-metadata]
[metabase.search.spec :as search.spec]
[metabase.search.test-util :as search.tu]
+ [metabase.search.util :as u.search]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util :as u]
@@ -57,6 +58,9 @@
:model/Card {} {:name "Projected Revenue" :collection_id col-id#}
:model/Card {} {:name "Employee Satisfaction" :collection_id col-id#}
:model/Card {} {:name "Projected Satisfaction" :collection_id col-id#}
+ :model/Card {} {:name "Organization Reference" :collection_id col-id#}
+ ;; this is to show off that our storage can give false positives (stemming etc)
+ :model/Card {} {:name "Nobody Expects Organi" :collection_id col-id#}
:model/Database {db-id# :id} {:name "Indexed Database"}
:model/Table {} {:name "Indexed Table", :db_id db-id#}]
(search.engine/reindex! :search.engine/appdb {:in-place? true})
@@ -147,14 +151,22 @@
(is (= 2 (index-hits "satisfaction -customer")))))))
(deftest phrase-test
- (with-index
- (with-fulltext-filtering
- ;; Less matches without an english dictionary
- (is (= #_2 3 (index-hits "projected")))
- (is (= 2 (index-hits "revenue")))
- (is (= #_1 2 (index-hits "projected revenue")))
- (testing "only sometimes do these occur sequentially in a phrase"
- (is (= 1 (index-hits "\"projected revenue\"")))))))
+ (mt/with-dynamic-fn-redefs [u.search/tsv-language (constantly "english")]
+ (with-index
+ (with-fulltext-filtering
+ ;; Less matches without an english dictionary
+ (is (= #_2 3 (index-hits "projected")))
+ (is (= 2 (index-hits "revenue")))
+ (is (= #_1 2 (index-hits "projected revenue")))
+ (testing "Simple dictionary allows searching for unstemmed words #60649"
+ ;; those two results are "Organization Reference" and "Nobody Expects Organi"
+ (is (= 2 (index-hits "organ")))
+ (is (= 2 (index-hits "organi"))))
+ (testing "We get false-positive for 'Organi' here"
+ ;; those two results are "Organization Reference" and "Nobody Expects Organi"
+ (is (= 2 (index-hits "organization"))))
+ (testing "only sometimes do these occur sequentially in a phrase"
+ (is (= 1 (index-hits "\"projected revenue\""))))))))
(defn ingest!
[model where-clause]
diff --git a/test/metabase/search/appdb/scoring_test.clj b/test/metabase/search/appdb/scoring_test.clj
index 88d56236071..ea33c5885bb 100644
--- a/test/metabase/search/appdb/scoring_test.clj
+++ b/test/metabase/search/appdb/scoring_test.clj
@@ -102,10 +102,10 @@
;; Due to stemming, we do not distinguish between exact matches and those that differ slightly.
(is (= [["card" 1 "orders"]
["card" 4 "order"]
- ;; We do not currently normalize the score based on the number of words in the vector / the coverage.
+ ;; We do not currently normalize the score based on the number of words in the vector / the coverage.
["card" 5 "orders, invoices, other stuff"]
["card" 6 "ordering"]
- ;; If the match is only in a secondary field, it is less preferred.
+ ;; If the match is only in a secondary field, it is less preferred.
["card" 3 "classified"]]
(search-results :text "order"))))
:h2