Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Disable typo tolerance explicitly only for CJK locales.

Kishore Nalland5112f72cb53
diff --git a/src/index.cpp b/src/index.cpp
index 21a80cad..a24e6cb7 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -3537,8 +3537,8 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
int64_t field_num_typos = (the_field.orig_index < num_typos.size()) ? num_typos[the_field.orig_index] : num_typos[0];
auto& locale = search_schema.at(the_field.name).locale;
- if(locale != "" && locale != "en" && locale != "th" && !Tokenizer::is_cyrillic(locale)) {
- // disable fuzzy trie traversal for non-english locales
+ if(locale != "" && (locale == "zh" || locale == "ko" || locale == "ja")) {
+ // disable fuzzy trie traversal for CJK locales
field_num_typos = 0;
}
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index c4db31a1..c3d34afd 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -820,6 +820,27 @@ TEST_F(CollectionLocaleTest, SearchOnArabicTextWithTypo) {
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
}
+TEST_F(CollectionLocaleTest, SearchInGermanLocaleShouldBeTypoTolerant) {
+ nlohmann::json coll_json = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title_de", "type": "string", "locale": "de"}
+ ]
+ })"_json;
+
+ auto coll1 = collectionManager.create_collection(coll_json).get();
+
+ nlohmann::json doc;
+ doc["title_de"] = "mülltonne";
+ doc["title_en"] = "trash bin";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("mulltonne", {"title_de"}, "", {}, {},
+ {2}, 10, 1, FREQUENCY, {true}, 1).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+}
+
/*
TEST_F(CollectionLocaleTest, TranslitPad) {
UErrorCode translit_status = U_ZERO_ERROR;