Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Normalize ascii tokens intermixed with non-english text.
Kishore Nallan359cdf15a57c
diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 127a3708..3ac192d0 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -1,4 +1,5 @@#include <sstream>+#include <algorithm>#include "tokenizer.h"bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_index, size_t& end_index) {@@ -33,6 +34,13 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde} else if(locale == "ko" && token == "·") {found_token = false;} else {++ if(std::isalnum(token[0]) && is_ascii_char(token[0])) {+ // normalize an ascii string+ std::transform(token.begin(), token.end(), token.begin(),+ [](unsigned char c){ return std::tolower(c); });+ }+found_token = true;token_index = token_counter++;}diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppindex 73c0b8cb..efb26943 100644--- a/test/collection_locale_test.cpp+++ b/test/collection_locale_test.cpp@@ -85,6 +85,49 @@ TEST_F(CollectionLocaleTest, SearchAgainstThaiText) {ASSERT_EQ("<mark>พกไฟ</mark>\nเสมอ", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());}+TEST_F(CollectionLocaleTest, SearchAgainstThaiTextExactMatch) {+ Collection* coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "th"),+ field("artist", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"ติดกับดักรายได้ปานกลาง", "Expected Result"},+ {"ข้อมูลรายคนหรือรายบริษัทในการเชื่อมโยงส่วนได้ส่วนเสีย", "Another Result"},+ };++ for (size_t i = 0; i < records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ std::vector<sort_by> sort_fields = { sort_by(sort_field_const::text_match, "DESC"), sort_by("points", "DESC") };+ auto results = coll1->search("รายได้",+ {"title"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY).get();++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());++ ASSERT_EQ("ติดกับดัก<mark>ราย</mark><mark>ได้</mark>ปานกลาง",+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ ASSERT_EQ("ข้อมูลรายคนหรือ<mark>ราย</mark>บริษัทในการเชื่อมโยงส่วน<mark>ได้</mark>ส่วนเสีย",+ results["hits"][1]["highlights"][0]["snippet"].get<std::string>());++}+TEST_F(CollectionLocaleTest, SearchAgainstKoreanText) {Collection *coll1;diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cppindex 681d12b1..bc94d793 100644--- a/test/tokenizer_test.cpp+++ b/test/tokenizer_test.cpp@@ -228,3 +228,12 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {ASSERT_EQ("な", tokens[2]);ASSERT_EQ("いぬ", tokens[3]);}++TEST(TokenizerTest, ShouldTokenizeLocaleTextWithEnglishText) {+ std::string tstr = "ผู้เขียนมีความสนใจเกี่ยวกับ Discrete Math และการคำนวณโดยทั่วไป";+ std::vector<std::string> ttokens;+ Tokenizer(tstr, true, false, "th").tokenize(ttokens);+ ASSERT_EQ(14, ttokens.size());+ ASSERT_EQ("discrete", ttokens[7]);+ ASSERT_EQ("math", ttokens[8]);+}