Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch โก ๐ โจ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Query token highlight threshold should consider unicode points.
Kishore Nallan2819c1d0a66c
diff --git a/src/collection.cpp b/src/collection.cppindex be392d8b..7d7d031b 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -3025,6 +3025,8 @@ bool Collection::handle_highlight_text(std::string& text, bool normalise, conststd::vector<std::string>& matched_tokens = highlight.matched_tokens.back();bool found_first_match = false;+ size_t text_len = Tokenizer::is_ascii_char(text[0]) ? text.size() : StringUtils::get_num_chars(text);+while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {if(use_word_tokenizer) {bool found_token = word_tokenizer.tokenize(raw_token);@@ -3053,8 +3055,8 @@ bool Collection::handle_highlight_text(std::string& text, bool normalise, const// Token might not appear in the best matched window, which is limited to a size of 10.// If field is marked to be highlighted fully, or field length exceeds snippet_threshold, we will// locate all tokens that appear in the query / query candidates- bool raw_token_found = !match_offset_found && (highlight_fully || text.size() < snippet_threshold * 6) &&- qtoken_leaves.find(raw_token) != qtoken_leaves.end();+ bool raw_token_found = !match_offset_found && (highlight_fully || text_len < snippet_threshold * 6) &&+ qtoken_leaves.find(raw_token) != qtoken_leaves.end();if (match_offset_found || raw_token_found) {if(qtoken_it != qtoken_leaves.end() && qtoken_it.value().is_prefix &&diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppindex c3d34afd..e9a2660c 100644--- a/test/collection_locale_test.cpp+++ b/test/collection_locale_test.cpp@@ -790,8 +790,6 @@ TEST_F(CollectionLocaleTest, SearchOnArabicText) {ASSERT_TRUE(coll1->add(doc.dump()).ok());auto results = coll1->search("ุฌููููุฉ", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}).get();- LOG(INFO) << results;-ASSERT_STREQ("<mark>ุฌูููุฉ</mark>",results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());}@@ -804,11 +802,11 @@ TEST_F(CollectionLocaleTest, SearchOnArabicTextWithTypo) {std::string title2 = "ุฏุงููู";nlohmann::json doc;- doc["title"] = "0";+ doc["id"] = "0";doc["title"] = "ูููุณ";ASSERT_TRUE(coll1->add(doc.dump()).ok());- doc["title"] = "1";+ doc["id"] = "1";doc["title"] = "ูููุงุฏ";ASSERT_TRUE(coll1->add(doc.dump()).ok());@@ -820,6 +818,23 @@ TEST_F(CollectionLocaleTest, SearchOnArabicTextWithTypo) {ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());}+TEST_F(CollectionLocaleTest, HighlightOfAllQueryTokensShouldConsiderUnicodePoints) {+ // For perfomance reasons, we highlight all query tokens in a text only on smaller text+ // Here, "small" threshold must be defined using unicode points and not raw string size.+ std::vector<field> fields = {field("title", field_types::STRING, true, false, true, ""),};+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["title"] = "ุฑุฌูุง ู ููู ุงุฌุชูุฏ ุงุฑุจุนูู ูููู ุซู ุฏุนุง ููู ูุณุชุฌุจ ูู ูุงุชู ุนูุณู ุงุจู ู ุฑูู ุนููู ุงูุณูุงู ูุดูู ุงููู ู ุง ูู ููู ููุณุงูู ุงูุฏุนุงุก ูู ูุชุทูุฑ ุนูุณู ูุตูู ุซู ";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto results = coll1->search("ูุฉ ุซู ุฏุนุง ููู ูุณุชุฌุจ ูู ูุฃุชู ุนูุณู ุงุจู ู ุฑูู ุนููู ุงูุณูุงู ูุดูู ุฅู", {"title"}, "", {}, {},+ {2}, 10, 1, FREQUENCY, {true}, 1).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(17, results["hits"][0]["highlights"][0]["matched_tokens"].size());+}+TEST_F(CollectionLocaleTest, SearchInGermanLocaleShouldBeTypoTolerant) {nlohmann::json coll_json = R"({"name": "coll1",