Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Ensure strings with both ASCII and non-ASCII entities are indexed properly.

kishorenc457bbcf46762
diff --git a/include/string_utils.h b/include/string_utils.h
index de261652..bb1cba6a 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -15,7 +15,7 @@ struct StringUtils {
StringUtils(): status(U_ZERO_ERROR) {
// transliterator(icu::Transliterator::createInstance("Latin-ASCII", UTRANS_FORWARD, status))
- cd = iconv_open("ascii//TRANSLIT", "UTF-8");
+ cd = iconv_open("ASCII//TRANSLIT", "UTF-8");
}
~StringUtils() {
@@ -156,6 +156,8 @@ struct StringUtils {
void unicode_normalize(std::string& str) const;
+ void unicode_normalize2(std::string& str) const;
+
/* https://stackoverflow.com/a/34571089/131050 */
static std::string base64_encode(const std::string &in) {
std::string out;
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
index 13e48bb4..d9b4c916 100644
--- a/src/string_utils.cpp
+++ b/src/string_utils.cpp
@@ -5,7 +5,8 @@ std::string lower_and_no_special_chars(const std::string & str) {
std::stringstream ss;
for(const auto c: str) {
- bool should_remove = (!std::isalnum(c) && (int)(c) >= 0);
+ bool should_remove = ( (int)(c) >= 0 && // check for ASCII range
+ !std::isalnum(c) );
if(!should_remove) {
ss << (char) std::tolower(c);
}
@@ -14,7 +15,41 @@ std::string lower_and_no_special_chars(const std::string & str) {
return ss.str();
}
-void StringUtils::unicode_normalize(std::string& str) const {
+void StringUtils::unicode_normalize(std::string & str) const {
+ std::stringstream out;
+
+ for (char *s = &str[0]; *s;) {
+ char inbuf[5];
+ char *p = inbuf;
+ *p++ = *s++;
+ if ((*s & 0xC0) == 0x80) *p++ = *s++;
+ if ((*s & 0xC0) == 0x80) *p++ = *s++;
+ if ((*s & 0xC0) == 0x80) *p++ = *s++;
+ *p = 0;
+ size_t insize = (p - &inbuf[0]);
+
+ char outbuf[5] = {};
+ size_t outsize = sizeof(outbuf);
+ char *outptr = outbuf;
+ char *inptr = inbuf;
+
+ //printf("[%s]\n", inbuf);
+
+ errno = 0;
+ iconv(cd, &inptr, &insize, &outptr, &outsize);
+
+ if(errno == EILSEQ) {
+ // symbol cannot be represented as ASCII, so write the original symbol
+ out << inbuf;
+ } else {
+ out << outbuf;
+ }
+ }
+
+ str.assign(lower_and_no_special_chars(out.str()));
+}
+
+void StringUtils::unicode_normalize2(std::string& str) const {
size_t outbuflen = str.length() * 2;
char output[outbuflen];
char *outptr = output;
diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cpp
index b7df56aa..de8d9fb8 100644
--- a/test/string_utils_test.cpp
+++ b/test/string_utils_test.cpp
@@ -18,9 +18,9 @@ TEST(StringUtilsTest, ShouldNormalizeString) {
string_utils.unicode_normalize(alphanum_specialchars);
ASSERT_STREQ("aa12zzwr", alphanum_specialchars.c_str());
- std::string alphanum_unicodechars = "abcÅà123ß";
+ std::string alphanum_unicodechars = "abcÅà123ß12";
string_utils.unicode_normalize(alphanum_unicodechars);
- ASSERT_STREQ("abcaa123ss", alphanum_unicodechars.c_str());
+ ASSERT_STREQ("abcaa123ss12", alphanum_unicodechars.c_str());
std::string tamil_unicodechars = "தமிழ் நாடு";
string_utils.unicode_normalize(tamil_unicodechars);
@@ -29,6 +29,12 @@ TEST(StringUtilsTest, ShouldNormalizeString) {
std::string chinese_unicodechars = "你好吗";
string_utils.unicode_normalize(chinese_unicodechars);
ASSERT_STREQ("你好吗", chinese_unicodechars.c_str());
+
+ std::string mixed_unicodechars = "çн தமிழ் நாடு so...";
+ string_utils.unicode_normalize(mixed_unicodechars);
+ ASSERT_STREQ("cнதமிழ்நாடுso", mixed_unicodechars.c_str());
+
+ // Any-Latin; Latin-ASCII; Lower()
}
TEST(StringUtilsTest, ShouldJoinString) {