Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add Tokenize test. (#756)

Harpreet Sangar9651c1abdf1f
diff --git a/include/string_utils.h b/include/string_utils.h
index 5a793b65..89592ad3 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -322,5 +322,5 @@ struct StringUtils {
static size_t get_num_chars(const std::string& text);
- static Option<bool> tokenize(std::string filter_query, std::queue<std::string>& tokens);
+ static Option<bool> tokenize_filter_query(const std::string& filter_query, std::queue<std::string>& tokens);
};
diff --git a/src/field.cpp b/src/field.cpp
index f9610a23..4e43cc59 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -414,7 +414,7 @@ Option<bool> filter::parse_filter_query(const std::string& filter_query,
}
std::queue<std::string> tokens;
- Option<bool> tokenize_op = StringUtils::tokenize(filter_query, tokens);
+ Option<bool> tokenize_op = StringUtils::tokenize_filter_query(filter_query, tokens);
if (!tokenize_op.ok()) {
return tokenize_op;
}
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
index 62ee8a24..a4374b65 100644
--- a/src/string_utils.cpp
+++ b/src/string_utils.cpp
@@ -345,7 +345,7 @@ size_t StringUtils::get_num_chars(const std::string& s) {
return j;
}
-Option<bool> StringUtils::tokenize(std::string filter_query, std::queue<std::string>& tokens) {
+Option<bool> StringUtils::tokenize_filter_query(const std::string& filter_query, std::queue<std::string>& tokens) {
auto size = filter_query.size();
for (auto i = 0; i < size;) {
auto c = filter_query[i];
@@ -401,7 +401,9 @@ Option<bool> StringUtils::tokenize(std::string filter_query, std::queue<std::str
} while (i < size && (inBacktick || is_geo_value ||
(c != '(' && c != ')' && !(c == '&' && filter_query[i + 1] == '&') &&
!(c == '|' && filter_query[i + 1] == '|'))));
- tokens.push(ss.str());
+ auto token = ss.str();
+ trim(token);
+ tokens.push(token);
}
}
return Option<bool>(true);
diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cpp
index 4ba2af73..8f296b6f 100644
--- a/test/string_utils_test.cpp
+++ b/test/string_utils_test.cpp
@@ -309,3 +309,51 @@ TEST(StringUtilsTest, ContainsWord) {
ASSERT_FALSE(StringUtils::contains_word("foobar baz", "bar baz"));
ASSERT_FALSE(StringUtils::contains_word("baz foobar", "foo"));
}
+
+void tokenizeTestHelper(const std::string& filter_query, const std::vector<std::string>& tokenList) {
+ std::queue<std::string> tokenizeOutput;
+ auto tokenize_op = StringUtils::tokenize_filter_query(filter_query, tokenizeOutput);
+ ASSERT_TRUE(tokenize_op.ok());
+ for (auto const& token: tokenList) {
+ ASSERT_EQ(token, tokenizeOutput.front());
+ tokenizeOutput.pop();
+ }
+ ASSERT_TRUE(tokenizeOutput.empty());
+}
+
+TEST(StringUtilsTest, TokenizeFilterQuery) {
+ std::string filter_query;
+ std::vector<std::string> tokenList;
+
+ filter_query = "name: Steve Smith";
+ tokenList = {"name: Steve Smith"};
+ tokenizeTestHelper(filter_query, tokenList);
+
+ filter_query = "name: `Toccata & Fugue`";
+ tokenList = {"name: `Toccata & Fugue`"};
+ tokenizeTestHelper(filter_query, tokenList);
+
+ filter_query = "name: [Steve Smith, `Jack & Jill`]";
+ tokenList = {"name: [Steve Smith, `Jack & Jill`]"};
+ tokenizeTestHelper(filter_query, tokenList);
+
+ filter_query = "age:[10..100]";
+ tokenList = {"age:[10..100]"};
+ tokenizeTestHelper(filter_query, tokenList);
+
+ filter_query = "age:>20 && category:= [`Running Shoes, Men`, Sneaker]";
+ tokenList = {"age:>20", "&&", "category:= [`Running Shoes, Men`, Sneaker]"};
+ tokenizeTestHelper(filter_query, tokenList);
+
+ filter_query = "location:(48.906, 2.343, 5 mi)";
+ tokenList = {"location:(48.906, 2.343, 5 mi)"};
+ tokenizeTestHelper(filter_query, tokenList);
+
+ filter_query = "((age: <5 || age: >10) && category:= [shoes]) || is_curated: true";
+ tokenList = {"(", "(", "age: <5", "||", "age: >10", ")", "&&", "category:= [shoes]", ")", "||", "is_curated: true"};
+ tokenizeTestHelper(filter_query, tokenList);
+
+ filter_query = "((age:<5||age:>10)&&location:(48.906,2.343,5mi))||tags:AT&T";
+ tokenList = {"(", "(", "age:<5", "||", "age:>10", ")", "&&", "location:(48.906,2.343,5mi)", ")", "||", "tags:AT&T"};
+ tokenizeTestHelper(filter_query, tokenList);
+}