Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch โก ๐ โจ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Arabic uses 2 bytes so 1 byte typos carry no meaning.
Kishore Nalland51ec38ef9ec
diff --git a/src/index.cpp b/src/index.cppindex 26e2a75c..f78bbb6c 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -5204,15 +5204,27 @@ inline uint32_t Index::next_suggestion2(const std::vector<tok_candidates>& tokensize_t token_size = token_candidates_vec[i].token.value.size();q = ldiv(q.quot, token_candidates_vec[i].candidates.size());const auto& candidate = token_candidates_vec[i].candidates[q.rem];+ size_t typo_cost = token_candidates_vec[i].cost;++ if (candidate.size() > 1 && !Tokenizer::is_ascii_char(candidate[0])) {+ icu::UnicodeString ustr = icu::UnicodeString::fromUTF8(candidate);+ auto code_point = ustr.char32At(0);+ if(code_point >= 0x600 && code_point <= 0x6ff) {+ // adjust typo cost for Arabic strings, since 1 byte difference makes no sense+ if(typo_cost == 1) {+ typo_cost = 2;+ }+ }+ }// we assume that toke was found via prefix search if candidate is longer than token's typo tolerancebool is_prefix_searched = token_candidates_vec[i].prefix_search &&- (candidate.size() > (token_size + token_candidates_vec[i].cost));+ (candidate.size() > (token_size + typo_cost));- size_t actual_cost = (2 * token_candidates_vec[i].cost) + uint32_t(is_prefix_searched);+ size_t actual_cost = (2 * typo_cost) + uint32_t(is_prefix_searched);total_cost += actual_cost;- query_suggestion[i] = token_t(i, candidate, is_prefix_searched, token_size, token_candidates_vec[i].cost);+ query_suggestion[i] = token_t(i, candidate, is_prefix_searched, token_size, typo_cost);uint64_t this_hash = StringUtils::hash_wy(query_suggestion[i].value.c_str(), query_suggestion[i].value.size());qhash = StringUtils::hash_combine(qhash, this_hash);diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppindex b12e68cb..c4db31a1 100644--- a/test/collection_locale_test.cpp+++ b/test/collection_locale_test.cpp@@ -796,6 +796,30 @@ TEST_F(CollectionLocaleTest, SearchOnArabicText) {results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());}+TEST_F(CollectionLocaleTest, SearchOnArabicTextWithTypo) {+ std::vector<field> fields = {field("title", field_types::STRING, true, false, true, ""),};+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();+ std::string q = "ุฏููู";+ std::string title1 = "ุณููู";+ std::string title2 = "ุฏุงููู";++ nlohmann::json doc;+ doc["title"] = "0";+ doc["title"] = "ูููุณ";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["title"] = "1";+ doc["title"] = "ูููุงุฏ";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto results = coll1->search("ูููุฏ", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {false}, 1,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "", 10).get();+ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());+}+/*TEST_F(CollectionLocaleTest, TranslitPad) {UErrorCode translit_status = U_ZERO_ERROR;