Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support remove_matched_tokens without filter_by.
Kishore Nallancfc402a92021
diff --git a/include/index.h b/include/index.hindex 32fbf52b..af1bf097 100644--- a/include/index.h+++ b/include/index.h@@ -83,7 +83,7 @@ struct override_t {std::vector<drop_hit_t> drop_hits;std::string filter_by;- bool remove_matched_tokens = true;+ bool remove_matched_tokens = false;override_t() = default;@@ -101,8 +101,9 @@ struct override_t {}if(override_json.count("includes") == 0 && override_json.count("excludes") == 0 &&- override_json.count("filter_by") == 0) {- return Option<bool>(400, "Must contain one of:`includes`, `excludes`, `filter_by`.");+ override_json.count("filter_by") == 0 && override_json.count("remove_matched_tokens") == 0) {+ return Option<bool>(400, "Must contain one of:`includes`, `excludes`, "+ "`filter_by`, `remove_matched_tokens`.");}if(override_json.count("includes") != 0) {@@ -164,8 +165,6 @@ struct override_t {if (!override_json["remove_matched_tokens"].is_boolean()) {return Option<bool>(400, "The `remove_matched_tokens` must be a boolean.");}-- override.remove_matched_tokens = override_json["remove_matched_tokens"].get<bool>();}if(!id.empty()) {@@ -200,6 +199,12 @@ struct override_t {override.filter_by = override_json["filter_by"].get<std::string>();}+ if(override_json.count("remove_matched_tokens") != 0) {+ override.remove_matched_tokens = override_json["remove_matched_tokens"].get<bool>();+ } else {+ override.remove_matched_tokens = (override_json.count("filter_by") != 0);+ }+// we have to also detect if it is a dynamic query rulesize_t i = 0;while(i < override.rule.query.size()) {diff --git a/src/collection.cpp b/src/collection.cppindex 1b00130a..de964f9e 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -451,6 +451,16 @@ void Collection::curate_results(string& actual_query, bool enable_overrides, booinclude_ids[hit.position].push_back(seq_id);}}++ if(override.remove_matched_tokens) {+ StringUtils::replace_all(query, override.rule.query, "");+ StringUtils::trim(query);+ if(query.empty()) {+ query = "*";+ }++ actual_query = query;+ }}}}diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex 31042182..86193ea8 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -263,7 +263,7 @@ TEST_F(CollectionOverrideTest, OverrideJSONValidation) {parse_op = override_t::parse(include_json2, "", override2);ASSERT_FALSE(parse_op.ok());- ASSERT_STREQ("Must contain one of:`includes`, `excludes`, `filter_by`.", parse_op.error().c_str());+ ASSERT_STREQ("Must contain one of:`includes`, `excludes`, `filter_by`, `remove_matched_tokens`.", parse_op.error().c_str());include_json2["includes"] = nlohmann::json::array();include_json2["includes"][0] = 100;@@ -1648,3 +1648,100 @@ TEST_F(CollectionOverrideTest, StaticFilterWithAndWithoutQueryStringMutation) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionOverrideTest, DynamicFilteringWithJustRemoveTokens) {+ Collection* coll1;++ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("category", field_types::STRING, true),+ field("brand", field_types::STRING, true),+ field("points", field_types::INT32, false)};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Amazing Shoes";+ doc1["category"] = "shoes";+ doc1["brand"] = "Nike";+ doc1["points"] = 3;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "Track Gym";+ doc2["category"] = "shoes";+ doc2["brand"] = "Adidas";+ doc2["points"] = 5;++ nlohmann::json doc3;+ doc3["id"] = "2";+ doc3["name"] = "Running Shoes";+ doc3["category"] = "sports";+ doc3["brand"] = "Nike";+ doc3["points"] = 5;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());++ std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC"), sort_by("points", "DESC")};++ auto results = coll1->search("all", {"name", "category", "brand"}, "",+ {}, sort_fields, {0, 0, 0}, 10).get();++ ASSERT_EQ(0, results["hits"].size());++ // with override, we return all records++ nlohmann::json override_json = {+ {"id", "match-all"},+ {+ "rule", {+ {"query", "all"},+ {"match", override_t::MATCH_EXACT}+ }+ },+ {"remove_matched_tokens", true}+ };++ override_t override;+ auto op = override_t::parse(override_json, "match-all", override);+ ASSERT_TRUE(op.ok());+ coll1->add_override(override);++ results = coll1->search("all", {"name", "category", "brand"}, "",+ {}, sort_fields, {0, 0, 0}, 10).get();++ ASSERT_EQ(3, results["hits"].size());++ results = coll1->search("really amazing shoes", {"name", "category", "brand"}, "",+ {}, sort_fields, {0, 0, 0}, 0).get();+ ASSERT_EQ(0, results["hits"].size());++ // with contains+ override_json = {+ {"id", "remove-some-tokens"},+ {+ "rule", {+ {"query", "really"},+ {"match", override_t::MATCH_CONTAINS}+ }+ },+ {"remove_matched_tokens", true}+ };++ override_t override2;+ op = override_t::parse(override_json, "remove-some-tokens", override2);+ ASSERT_TRUE(op.ok());+ coll1->add_override(override2);++ results = coll1->search("really amazing shoes", {"name", "category", "brand"}, "",+ {}, sort_fields, {0, 0, 0}, 1).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());++ collectionManager.drop_collection("coll1");+}