Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Ensure strings with both ASCII and non-ASCII entities are indexed properly.
kishorenc457bbcf46762
diff --git a/include/string_utils.h b/include/string_utils.hindex de261652..bb1cba6a 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -15,7 +15,7 @@ struct StringUtils {StringUtils(): status(U_ZERO_ERROR) {// transliterator(icu::Transliterator::createInstance("Latin-ASCII", UTRANS_FORWARD, status))- cd = iconv_open("ascii//TRANSLIT", "UTF-8");+ cd = iconv_open("ASCII//TRANSLIT", "UTF-8");}~StringUtils() {@@ -156,6 +156,8 @@ struct StringUtils {void unicode_normalize(std::string& str) const;+ void unicode_normalize2(std::string& str) const;+/* https://stackoverflow.com/a/34571089/131050 */static std::string base64_encode(const std::string &in) {std::string out;diff --git a/src/string_utils.cpp b/src/string_utils.cppindex 13e48bb4..d9b4c916 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -5,7 +5,8 @@ std::string lower_and_no_special_chars(const std::string & str) {std::stringstream ss;for(const auto c: str) {- bool should_remove = (!std::isalnum(c) && (int)(c) >= 0);+ bool should_remove = ( (int)(c) >= 0 && // check for ASCII range+ !std::isalnum(c) );if(!should_remove) {ss << (char) std::tolower(c);}@@ -14,7 +15,41 @@ std::string lower_and_no_special_chars(const std::string & str) {return ss.str();}-void StringUtils::unicode_normalize(std::string& str) const {+void StringUtils::unicode_normalize(std::string & str) const {+ std::stringstream out;++ for (char *s = &str[0]; *s;) {+ char inbuf[5];+ char *p = inbuf;+ *p++ = *s++;+ if ((*s & 0xC0) == 0x80) *p++ = *s++;+ if ((*s & 0xC0) == 0x80) *p++ = *s++;+ if ((*s & 0xC0) == 0x80) *p++ = *s++;+ *p = 0;+ size_t insize = (p - &inbuf[0]);++ char outbuf[5] = {};+ size_t outsize = sizeof(outbuf);+ char *outptr = outbuf;+ char *inptr = inbuf;++ //printf("[%s]\n", inbuf);++ errno = 0;+ iconv(cd, &inptr, &insize, &outptr, &outsize);++ if(errno == EILSEQ) {+ // symbol cannot be represented as ASCII, so write the original symbol+ out << inbuf;+ } else {+ out << outbuf;+ }+ }++ str.assign(lower_and_no_special_chars(out.str()));+}++void StringUtils::unicode_normalize2(std::string& str) const {size_t outbuflen = str.length() * 2;char output[outbuflen];char *outptr = output;diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cppindex b7df56aa..de8d9fb8 100644--- a/test/string_utils_test.cpp+++ b/test/string_utils_test.cpp@@ -18,9 +18,9 @@ TEST(StringUtilsTest, ShouldNormalizeString) {string_utils.unicode_normalize(alphanum_specialchars);ASSERT_STREQ("aa12zzwr", alphanum_specialchars.c_str());- std::string alphanum_unicodechars = "abcÅà123ß";+ std::string alphanum_unicodechars = "abcÅà123ß12";string_utils.unicode_normalize(alphanum_unicodechars);- ASSERT_STREQ("abcaa123ss", alphanum_unicodechars.c_str());+ ASSERT_STREQ("abcaa123ss12", alphanum_unicodechars.c_str());std::string tamil_unicodechars = "தமிழ் நாடு";string_utils.unicode_normalize(tamil_unicodechars);@@ -29,6 +29,12 @@ TEST(StringUtilsTest, ShouldNormalizeString) {std::string chinese_unicodechars = "你好吗";string_utils.unicode_normalize(chinese_unicodechars);ASSERT_STREQ("你好吗", chinese_unicodechars.c_str());++ std::string mixed_unicodechars = "çн தமிழ் நாடு so...";+ string_utils.unicode_normalize(mixed_unicodechars);+ ASSERT_STREQ("cнதமிழ்நாடுso", mixed_unicodechars.c_str());++ // Any-Latin; Latin-ASCII; Lower()}TEST(StringUtilsTest, ShouldJoinString) {