Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Normalize ascii tokens intermixed with non-english text.

Kishore Nallan8d6742fc6d71
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 127a3708..3ac192d0 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -1,4 +1,5 @@
#include <sstream>
+#include <algorithm>
#include "tokenizer.h"
bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_index, size_t& end_index) {
@@ -33,6 +34,13 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
} else if(locale == "ko" && token == "·") {
found_token = false;
} else {
+
+ if(std::isalnum(token[0]) && is_ascii_char(token[0])) {
+ // normalize an ascii string
+ std::transform(token.begin(), token.end(), token.begin(),
+ [](unsigned char c){ return std::tolower(c); });
+ }
+
found_token = true;
token_index = token_counter++;
}
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index 73c0b8cb..efb26943 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -85,6 +85,49 @@ TEST_F(CollectionLocaleTest, SearchAgainstThaiText) {
ASSERT_EQ("<mark>พกไฟ</mark>\nเสมอ", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
}
+TEST_F(CollectionLocaleTest, SearchAgainstThaiTextExactMatch) {
+ Collection* coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "th"),
+ field("artist", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"ติดกับดักรายได้ปานกลาง", "Expected Result"},
+ {"ข้อมูลรายคนหรือรายบริษัทในการเชื่อมโยงส่วนได้ส่วนเสีย", "Another Result"},
+ };
+
+ for (size_t i = 0; i < records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ std::vector<sort_by> sort_fields = { sort_by(sort_field_const::text_match, "DESC"), sort_by("points", "DESC") };
+ auto results = coll1->search("รายได้",
+ {"title"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(2, results["found"].get<size_t>());
+ ASSERT_EQ(2, results["hits"].size());
+
+ ASSERT_EQ("ติดกับดัก<mark>ราย</mark><mark>ได้</mark>ปานกลาง",
+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
+ ASSERT_EQ("ข้อมูลรายคนหรือ<mark>ราย</mark>บริษัทในการเชื่อมโยงส่วน<mark>ได้</mark>ส่วนเสีย",
+ results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
+
+}
+
TEST_F(CollectionLocaleTest, SearchAgainstKoreanText) {
Collection *coll1;
diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cpp
index 681d12b1..bc94d793 100644
--- a/test/tokenizer_test.cpp
+++ b/test/tokenizer_test.cpp
@@ -228,3 +228,12 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
ASSERT_EQ("な", tokens[2]);
ASSERT_EQ("いぬ", tokens[3]);
}
+
+TEST(TokenizerTest, ShouldTokenizeLocaleTextWithEnglishText) {
+ std::string tstr = "ผู้เขียนมีความสนใจเกี่ยวกับ Discrete Math และการคำนวณโดยทั่วไป";
+ std::vector<std::string> ttokens;
+ Tokenizer(tstr, true, false, "th").tokenize(ttokens);
+ ASSERT_EQ(14, ttokens.size());
+ ASSERT_EQ("discrete", ttokens[7]);
+ ASSERT_EQ("math", ttokens[8]);
+}