Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch โšก ๐Ÿ” โœจ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Arabic uses 2 bytes so 1 byte typos carry no meaning.

Kishore Nalland51ec38ef9ec
diff --git a/src/index.cpp b/src/index.cpp
index 26e2a75c..f78bbb6c 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -5204,15 +5204,27 @@ inline uint32_t Index::next_suggestion2(const std::vector<tok_candidates>& token
size_t token_size = token_candidates_vec[i].token.value.size();
q = ldiv(q.quot, token_candidates_vec[i].candidates.size());
const auto& candidate = token_candidates_vec[i].candidates[q.rem];
+ size_t typo_cost = token_candidates_vec[i].cost;
+
+ if (candidate.size() > 1 && !Tokenizer::is_ascii_char(candidate[0])) {
+ icu::UnicodeString ustr = icu::UnicodeString::fromUTF8(candidate);
+ auto code_point = ustr.char32At(0);
+ if(code_point >= 0x600 && code_point <= 0x6ff) {
+ // adjust typo cost for Arabic strings, since 1 byte difference makes no sense
+ if(typo_cost == 1) {
+ typo_cost = 2;
+ }
+ }
+ }
// we assume that toke was found via prefix search if candidate is longer than token's typo tolerance
bool is_prefix_searched = token_candidates_vec[i].prefix_search &&
- (candidate.size() > (token_size + token_candidates_vec[i].cost));
+ (candidate.size() > (token_size + typo_cost));
- size_t actual_cost = (2 * token_candidates_vec[i].cost) + uint32_t(is_prefix_searched);
+ size_t actual_cost = (2 * typo_cost) + uint32_t(is_prefix_searched);
total_cost += actual_cost;
- query_suggestion[i] = token_t(i, candidate, is_prefix_searched, token_size, token_candidates_vec[i].cost);
+ query_suggestion[i] = token_t(i, candidate, is_prefix_searched, token_size, typo_cost);
uint64_t this_hash = StringUtils::hash_wy(query_suggestion[i].value.c_str(), query_suggestion[i].value.size());
qhash = StringUtils::hash_combine(qhash, this_hash);
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index b12e68cb..c4db31a1 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -796,6 +796,30 @@ TEST_F(CollectionLocaleTest, SearchOnArabicText) {
results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
}
+TEST_F(CollectionLocaleTest, SearchOnArabicTextWithTypo) {
+ std::vector<field> fields = {field("title", field_types::STRING, true, false, true, ""),};
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+ std::string q = "ุฏูˆู†ูŠ";
+ std::string title1 = "ุณูˆู†ูŠ";
+ std::string title2 = "ุฏุงูˆู†ูŠ";
+
+ nlohmann::json doc;
+ doc["title"] = "0";
+ doc["title"] = "ูŠู†ูˆุณ";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["title"] = "1";
+ doc["title"] = "ูŠู†ูˆุงุฏ";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("ูŠู†ูˆุฏ", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {false}, 1,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "", 10).get();
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+}
+
/*
TEST_F(CollectionLocaleTest, TranslitPad) {
UErrorCode translit_status = U_ZERO_ERROR;