Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add Tokenize test. (#756)
Harpreet Sangar9651c1abdf1f
diff --git a/include/string_utils.h b/include/string_utils.hindex 5a793b65..89592ad3 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -322,5 +322,5 @@ struct StringUtils {static size_t get_num_chars(const std::string& text);- static Option<bool> tokenize(std::string filter_query, std::queue<std::string>& tokens);+ static Option<bool> tokenize_filter_query(const std::string& filter_query, std::queue<std::string>& tokens);};diff --git a/src/field.cpp b/src/field.cppindex f9610a23..4e43cc59 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -414,7 +414,7 @@ Option<bool> filter::parse_filter_query(const std::string& filter_query,}std::queue<std::string> tokens;- Option<bool> tokenize_op = StringUtils::tokenize(filter_query, tokens);+ Option<bool> tokenize_op = StringUtils::tokenize_filter_query(filter_query, tokens);if (!tokenize_op.ok()) {return tokenize_op;}diff --git a/src/string_utils.cpp b/src/string_utils.cppindex 62ee8a24..a4374b65 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -345,7 +345,7 @@ size_t StringUtils::get_num_chars(const std::string& s) {return j;}-Option<bool> StringUtils::tokenize(std::string filter_query, std::queue<std::string>& tokens) {+Option<bool> StringUtils::tokenize_filter_query(const std::string& filter_query, std::queue<std::string>& tokens) {auto size = filter_query.size();for (auto i = 0; i < size;) {auto c = filter_query[i];@@ -401,7 +401,9 @@ Option<bool> StringUtils::tokenize(std::string filter_query, std::queue<std::str} while (i < size && (inBacktick || is_geo_value ||(c != '(' && c != ')' && !(c == '&' && filter_query[i + 1] == '&') &&!(c == '|' && filter_query[i + 1] == '|'))));- tokens.push(ss.str());+ auto token = ss.str();+ trim(token);+ tokens.push(token);}}return Option<bool>(true);diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cppindex 4ba2af73..8f296b6f 100644--- a/test/string_utils_test.cpp+++ b/test/string_utils_test.cpp@@ -309,3 +309,51 @@ TEST(StringUtilsTest, ContainsWord) {ASSERT_FALSE(StringUtils::contains_word("foobar baz", "bar baz"));ASSERT_FALSE(StringUtils::contains_word("baz foobar", "foo"));}++void tokenizeTestHelper(const std::string& filter_query, const std::vector<std::string>& tokenList) {+ std::queue<std::string> tokenizeOutput;+ auto tokenize_op = StringUtils::tokenize_filter_query(filter_query, tokenizeOutput);+ ASSERT_TRUE(tokenize_op.ok());+ for (auto const& token: tokenList) {+ ASSERT_EQ(token, tokenizeOutput.front());+ tokenizeOutput.pop();+ }+ ASSERT_TRUE(tokenizeOutput.empty());+}++TEST(StringUtilsTest, TokenizeFilterQuery) {+ std::string filter_query;+ std::vector<std::string> tokenList;++ filter_query = "name: Steve Smith";+ tokenList = {"name: Steve Smith"};+ tokenizeTestHelper(filter_query, tokenList);++ filter_query = "name: `Toccata & Fugue`";+ tokenList = {"name: `Toccata & Fugue`"};+ tokenizeTestHelper(filter_query, tokenList);++ filter_query = "name: [Steve Smith, `Jack & Jill`]";+ tokenList = {"name: [Steve Smith, `Jack & Jill`]"};+ tokenizeTestHelper(filter_query, tokenList);++ filter_query = "age:[10..100]";+ tokenList = {"age:[10..100]"};+ tokenizeTestHelper(filter_query, tokenList);++ filter_query = "age:>20 && category:= [`Running Shoes, Men`, Sneaker]";+ tokenList = {"age:>20", "&&", "category:= [`Running Shoes, Men`, Sneaker]"};+ tokenizeTestHelper(filter_query, tokenList);++ filter_query = "location:(48.906, 2.343, 5 mi)";+ tokenList = {"location:(48.906, 2.343, 5 mi)"};+ tokenizeTestHelper(filter_query, tokenList);++ filter_query = "((age: <5 || age: >10) && category:= [shoes]) || is_curated: true";+ tokenList = {"(", "(", "age: <5", "||", "age: >10", ")", "&&", "category:= [shoes]", ")", "||", "is_curated: true"};+ tokenizeTestHelper(filter_query, tokenList);++ filter_query = "((age:<5||age:>10)&&location:(48.906,2.343,5mi))||tags:AT&T";+ tokenList = {"(", "(", "age:<5", "||", "age:>10", ")", "&&", "location:(48.906,2.343,5mi)", ")", "||", "tags:AT&T"};+ tokenizeTestHelper(filter_query, tokenList);+}