Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch โšก ๐Ÿ” โœจ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Query token highlight threshold should consider unicode points.

Kishore Nallan2819c1d0a66c
diff --git a/src/collection.cpp b/src/collection.cpp
index be392d8b..7d7d031b 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -3025,6 +3025,8 @@ bool Collection::handle_highlight_text(std::string& text, bool normalise, const
std::vector<std::string>& matched_tokens = highlight.matched_tokens.back();
bool found_first_match = false;
+ size_t text_len = Tokenizer::is_ascii_char(text[0]) ? text.size() : StringUtils::get_num_chars(text);
+
while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {
if(use_word_tokenizer) {
bool found_token = word_tokenizer.tokenize(raw_token);
@@ -3053,8 +3055,8 @@ bool Collection::handle_highlight_text(std::string& text, bool normalise, const
// Token might not appear in the best matched window, which is limited to a size of 10.
// If field is marked to be highlighted fully, or field length exceeds snippet_threshold, we will
// locate all tokens that appear in the query / query candidates
- bool raw_token_found = !match_offset_found && (highlight_fully || text.size() < snippet_threshold * 6) &&
- qtoken_leaves.find(raw_token) != qtoken_leaves.end();
+ bool raw_token_found = !match_offset_found && (highlight_fully || text_len < snippet_threshold * 6) &&
+ qtoken_leaves.find(raw_token) != qtoken_leaves.end();
if (match_offset_found || raw_token_found) {
if(qtoken_it != qtoken_leaves.end() && qtoken_it.value().is_prefix &&
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index c3d34afd..e9a2660c 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -790,8 +790,6 @@ TEST_F(CollectionLocaleTest, SearchOnArabicText) {
ASSERT_TRUE(coll1->add(doc.dump()).ok());
auto results = coll1->search("ุฌูˆู‡ูŠู†ุฉ", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}).get();
- LOG(INFO) << results;
-
ASSERT_STREQ("<mark>ุฌู‡ูŠู†ุฉ</mark>",
results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
}
@@ -804,11 +802,11 @@ TEST_F(CollectionLocaleTest, SearchOnArabicTextWithTypo) {
std::string title2 = "ุฏุงูˆู†ูŠ";
nlohmann::json doc;
- doc["title"] = "0";
+ doc["id"] = "0";
doc["title"] = "ูŠู†ูˆุณ";
ASSERT_TRUE(coll1->add(doc.dump()).ok());
- doc["title"] = "1";
+ doc["id"] = "1";
doc["title"] = "ูŠู†ูˆุงุฏ";
ASSERT_TRUE(coll1->add(doc.dump()).ok());
@@ -820,6 +818,23 @@ TEST_F(CollectionLocaleTest, SearchOnArabicTextWithTypo) {
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
}
+TEST_F(CollectionLocaleTest, HighlightOfAllQueryTokensShouldConsiderUnicodePoints) {
+ // For perfomance reasons, we highlight all query tokens in a text only on smaller text
+ // Here, "small" threshold must be defined using unicode points and not raw string size.
+ std::vector<field> fields = {field("title", field_types::STRING, true, false, true, ""),};
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = "ุฑุฌู„ุง ู…ู†ู‡ู… ุงุฌุชู‡ุฏ ุงุฑุจุนูŠู† ู„ูŠู„ู‡ ุซู… ุฏุนุง ูู„ู… ูŠุณุชุฌุจ ู„ู‡ ูุงุชูŠ ุนูŠุณูŠ ุงุจู† ู…ุฑูŠู… ุนู„ูŠู‡ ุงู„ุณู„ุงู… ูŠุดูƒูˆ ุงู„ูŠู‡ ู…ุง ู‡ูˆ ููŠู‡ ูˆูŠุณุงู„ู‡ ุงู„ุฏุนุงุก ู„ู‡ ูุชุทู‡ุฑ ุนูŠุณูŠ ูˆุตู„ูŠ ุซู…";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("ู„ุฉ ุซู… ุฏุนุง ูู„ู… ูŠุณุชุฌุจ ู„ู‡ ูุฃุชู‰ ุนูŠุณู‰ ุงุจู† ู…ุฑูŠู… ุนู„ูŠู‡ ุงู„ุณู„ุงู… ูŠุดูƒูˆ ุฅู„", {"title"}, "", {}, {},
+ {2}, 10, 1, FREQUENCY, {true}, 1).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(17, results["hits"][0]["highlights"][0]["matched_tokens"].size());
+}
+
TEST_F(CollectionLocaleTest, SearchInGermanLocaleShouldBeTypoTolerant) {
nlohmann::json coll_json = R"({
"name": "coll1",