Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix cyrillic translit producing double quotes as output.

Kishore Nallana670b579f793
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index baa3132a..a4773161 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -128,6 +128,7 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
auto raw_text = unicode_text.tempSubStringBetween(start_pos, end_pos);
transliterator->transliterate(raw_text);
raw_text.toUTF8String(word);
+ StringUtils::replace_all(word, "\"", "");
} else if(normalize && locale == "th") {
UErrorCode errcode = U_ZERO_ERROR;
icu::UnicodeString src = unicode_text.tempSubStringBetween(start_pos, end_pos);
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index b58a1a03..fbb19a75 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -839,6 +839,34 @@ TEST_F(CollectionLocaleTest, SearchOnArabicTextWithTypo) {
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
}
+TEST_F(CollectionLocaleTest, SearchOnBulgarianText) {
+ std::vector<field> fields = {field("title", field_types::STRING, true, false, true, "bg"),};
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+ std::string title1 = "Сърце от любов";
+ std::string title2 = "Съблезъб тигър";
+ std::string title3 = "Сърна";
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = title1;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["id"] = "1";
+ doc["title"] = title2;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["id"] = "2";
+ doc["title"] = title3;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("Сърце", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "", 10).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+}
+
TEST_F(CollectionLocaleTest, HighlightOfAllQueryTokensShouldConsiderUnicodePoints) {
// For perfomance reasons, we highlight all query tokens in a text only on smaller text
// Here, "small" threshold must be defined using unicode points and not raw string size.