Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Tokenize/normalize override query properly.
Kishore Nallanc7a7df676b8e
diff --git a/include/override.h b/include/override.hindex a92c01c2..2900f866 100644--- a/include/override.h+++ b/include/override.h@@ -45,7 +45,12 @@ struct override_t {override_t() = default;- static Option<bool> parse(const nlohmann::json& override_json, const std::string& id, override_t& override);+ static Option<bool> parse(const nlohmann::json& override_json, const std::string& id,+ override_t& override,+ const std::string& locale = "",+ const std::vector<char>& symbols_to_index = {},+ const std::vector<char>& token_separators = {}+ );nlohmann::json to_json() const;};diff --git a/src/collection.cpp b/src/collection.cppindex 362088ef..458df719 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -643,10 +643,18 @@ void Collection::curate_results(string& actual_query, const string& filter_query}}- std::string query = actual_query;if(enable_overrides && !overrides.empty()) {- StringUtils::tolowercase(query);+ std::string query;++ if(actual_query == "*") {+ query = "*";+ } else {+ std::vector<std::string> tokens;+ Tokenizer tokenizer(actual_query, true, false, "", symbols_to_index, token_separators);+ tokenizer.tokenize(tokens);+ query = StringUtils::join(tokens, " ");+ }for(const auto& override_kv: overrides) {const auto& override = override_kv.second;diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 2155f2f5..1c7e912b 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1311,7 +1311,8 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectionfor(const auto & collection_override_json: collection_override_jsons) {nlohmann::json collection_override = nlohmann::json::parse(collection_override_json);override_t override;- auto parse_op = override_t::parse(collection_override, "", override);+ auto parse_op = override_t::parse(collection_override, "", override, "", collection->get_symbols_to_index(),+ collection->get_token_separators());if(parse_op.ok()) {collection->add_override(override, false);} else {diff --git a/src/core_api.cpp b/src/core_api.cppindex b67b6465..e3b17d6a 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -1399,7 +1399,9 @@ bool put_override(const std::shared_ptr<http_req>& req, const std::shared_ptr<ht}override_t override;- Option<bool> parse_op = override_t::parse(req_json, override_id, override);+ Option<bool> parse_op = override_t::parse(req_json, override_id, override, "",+ collection->get_symbols_to_index(),+ collection->get_token_separators());if(!parse_op.ok()) {res->set(parse_op.code(), parse_op.error());return false;diff --git a/src/override.cpp b/src/override.cppindex 96f96d38..572189d5 100644--- a/src/override.cpp+++ b/src/override.cpp@@ -2,7 +2,11 @@#include "override.h"#include "tokenizer.h"-Option<bool> override_t::parse(const nlohmann::json& override_json, const std::string& id, override_t& override) {+Option<bool> override_t::parse(const nlohmann::json& override_json, const std::string& id,+ override_t& override,+ const std::string& locale,+ const std::vector<char>& symbols_to_index,+ const std::vector<char>& token_separators) {if(!override_json.is_object()) {return Option<bool>(400, "Bad JSON.");}@@ -110,8 +114,14 @@ Option<bool> override_t::parse(const nlohmann::json& override_json, const std::soverride.rule.match = json_rule.count("match") == 0 ? "" : json_rule["match"].get<std::string>();if(!override.rule.query.empty()) {- override.rule.normalized_query = override.rule.query;- Tokenizer::normalize_ascii(override.rule.normalized_query);+ auto symbols = symbols_to_index;+ symbols.push_back('{');+ symbols.push_back('}');+ symbols.push_back('*');+ Tokenizer tokenizer(override.rule.query, true, false, locale, symbols, token_separators);+ std::vector<std::string> tokens;+ tokenizer.tokenize(tokens);+ override.rule.normalized_query = StringUtils::join(tokens, " ");}if(json_rule.count("filter_by") != 0) {diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex b602c777..f5b368ec 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -948,6 +948,50 @@ TEST_F(CollectionOverrideTest, RuleQueryMustBeCaseInsensitive) {ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());}+TEST_F(CollectionOverrideTest, RuleQueryWithAccentedChars) {+ Collection *coll1;++ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("color", field_types::STRING, false),+ field("points", field_types::INT32, false)};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Green";+ doc1["color"] = "Green";+ doc1["points"] = 30;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());++ std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };++ nlohmann::json override_json = R"({+ "id": "rule-1",+ "rule": {+ "query": "Grün",+ "match": "contains"+ },+ "filter_by":"color:green",+ "filter_curated_hits":true+ })"_json;++ override_t override_rule;+ auto op = override_t::parse(override_json, "rule-1", override_rule);+ ASSERT_TRUE(op.ok());+ coll1->add_override(override_rule);++ auto results = coll1->search("grün", {"name"}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+}+TEST_F(CollectionOverrideTest, WindowForRule) {Collection *coll1;@@ -1428,7 +1472,7 @@ TEST_F(CollectionOverrideTest, PinnedHitsWhenThereAreNotEnoughResults) {// multiple pinned hits specified, but query produces no result- auto results = coll_mul_fields->search("notfoundquery", {"title"}, "", {"starring"}, {}, {0}, 10, 1, FREQUENCY,+ auto results = coll_mul_fields->search("not-foundquery", {"title"}, "", {"starring"}, {}, {0}, 10, 1, FREQUENCY,{false}, Index::DROP_TOKENS_THRESHOLD,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,@@ -3158,3 +3202,80 @@ TEST_F(CollectionOverrideTest, DynamicFilteringWithPartialTokenMatch) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionOverrideTest, OverrideWithSymbolsToIndex) {+ Collection* coll1;++ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("category", field_types::STRING, true),};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "", static_cast<uint64_t>(std::time(nullptr)),+ "", {"-"}, {}).get();+ }++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Non-Stick";+ doc1["category"] = "Cookware";++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "NonStick";+ doc2["category"] = "Kitchen";++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());++ std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC")};++ auto results = coll1->search("non-stick", {"name"}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10).get();++ ASSERT_EQ(2, results["hits"].size());++ // with override, we return all records++ nlohmann::json override_json = {+ {"id", "ov-1"},+ {+ "rule", {+ {"query", "non-stick"},+ {"match", override_t::MATCH_EXACT}+ }+ },+ {"filter_by", "category:= Cookware"}+ };++ override_t override;+ auto op = override_t::parse(override_json, "ov-1", override, "", {'-'}, {});+ ASSERT_TRUE(op.ok());+ coll1->add_override(override);++ results = coll1->search("non-stick", {"name"}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());++ results = coll1->search("nonstick", {"name"}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10).get();++ ASSERT_EQ(2, results["hits"].size());++ collectionManager.drop_collection("coll1");+}+