Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Support remove_matched_tokens without filter_by.

Kishore Nallancfc402a92021
diff --git a/include/index.h b/include/index.h
index 32fbf52b..af1bf097 100644
--- a/include/index.h
+++ b/include/index.h
@@ -83,7 +83,7 @@ struct override_t {
std::vector<drop_hit_t> drop_hits;
std::string filter_by;
- bool remove_matched_tokens = true;
+ bool remove_matched_tokens = false;
override_t() = default;
@@ -101,8 +101,9 @@ struct override_t {
}
if(override_json.count("includes") == 0 && override_json.count("excludes") == 0 &&
- override_json.count("filter_by") == 0) {
- return Option<bool>(400, "Must contain one of:`includes`, `excludes`, `filter_by`.");
+ override_json.count("filter_by") == 0 && override_json.count("remove_matched_tokens") == 0) {
+ return Option<bool>(400, "Must contain one of:`includes`, `excludes`, "
+ "`filter_by`, `remove_matched_tokens`.");
}
if(override_json.count("includes") != 0) {
@@ -164,8 +165,6 @@ struct override_t {
if (!override_json["remove_matched_tokens"].is_boolean()) {
return Option<bool>(400, "The `remove_matched_tokens` must be a boolean.");
}
-
- override.remove_matched_tokens = override_json["remove_matched_tokens"].get<bool>();
}
if(!id.empty()) {
@@ -200,6 +199,12 @@ struct override_t {
override.filter_by = override_json["filter_by"].get<std::string>();
}
+ if(override_json.count("remove_matched_tokens") != 0) {
+ override.remove_matched_tokens = override_json["remove_matched_tokens"].get<bool>();
+ } else {
+ override.remove_matched_tokens = (override_json.count("filter_by") != 0);
+ }
+
// we have to also detect if it is a dynamic query rule
size_t i = 0;
while(i < override.rule.query.size()) {
diff --git a/src/collection.cpp b/src/collection.cpp
index 1b00130a..de964f9e 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -451,6 +451,16 @@ void Collection::curate_results(string& actual_query, bool enable_overrides, boo
include_ids[hit.position].push_back(seq_id);
}
}
+
+ if(override.remove_matched_tokens) {
+ StringUtils::replace_all(query, override.rule.query, "");
+ StringUtils::trim(query);
+ if(query.empty()) {
+ query = "*";
+ }
+
+ actual_query = query;
+ }
}
}
}
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index 31042182..86193ea8 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -263,7 +263,7 @@ TEST_F(CollectionOverrideTest, OverrideJSONValidation) {
parse_op = override_t::parse(include_json2, "", override2);
ASSERT_FALSE(parse_op.ok());
- ASSERT_STREQ("Must contain one of:`includes`, `excludes`, `filter_by`.", parse_op.error().c_str());
+ ASSERT_STREQ("Must contain one of:`includes`, `excludes`, `filter_by`, `remove_matched_tokens`.", parse_op.error().c_str());
include_json2["includes"] = nlohmann::json::array();
include_json2["includes"][0] = 100;
@@ -1648,3 +1648,100 @@ TEST_F(CollectionOverrideTest, StaticFilterWithAndWithoutQueryStringMutation) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionOverrideTest, DynamicFilteringWithJustRemoveTokens) {
+ Collection* coll1;
+
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("category", field_types::STRING, true),
+ field("brand", field_types::STRING, true),
+ field("points", field_types::INT32, false)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Amazing Shoes";
+ doc1["category"] = "shoes";
+ doc1["brand"] = "Nike";
+ doc1["points"] = 3;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "Track Gym";
+ doc2["category"] = "shoes";
+ doc2["brand"] = "Adidas";
+ doc2["points"] = 5;
+
+ nlohmann::json doc3;
+ doc3["id"] = "2";
+ doc3["name"] = "Running Shoes";
+ doc3["category"] = "sports";
+ doc3["brand"] = "Nike";
+ doc3["points"] = 5;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());
+
+ std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC"), sort_by("points", "DESC")};
+
+ auto results = coll1->search("all", {"name", "category", "brand"}, "",
+ {}, sort_fields, {0, 0, 0}, 10).get();
+
+ ASSERT_EQ(0, results["hits"].size());
+
+ // with override, we return all records
+
+ nlohmann::json override_json = {
+ {"id", "match-all"},
+ {
+ "rule", {
+ {"query", "all"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ },
+ {"remove_matched_tokens", true}
+ };
+
+ override_t override;
+ auto op = override_t::parse(override_json, "match-all", override);
+ ASSERT_TRUE(op.ok());
+ coll1->add_override(override);
+
+ results = coll1->search("all", {"name", "category", "brand"}, "",
+ {}, sort_fields, {0, 0, 0}, 10).get();
+
+ ASSERT_EQ(3, results["hits"].size());
+
+ results = coll1->search("really amazing shoes", {"name", "category", "brand"}, "",
+ {}, sort_fields, {0, 0, 0}, 0).get();
+ ASSERT_EQ(0, results["hits"].size());
+
+ // with contains
+ override_json = {
+ {"id", "remove-some-tokens"},
+ {
+ "rule", {
+ {"query", "really"},
+ {"match", override_t::MATCH_CONTAINS}
+ }
+ },
+ {"remove_matched_tokens", true}
+ };
+
+ override_t override2;
+ op = override_t::parse(override_json, "remove-some-tokens", override2);
+ ASSERT_TRUE(op.ok());
+ coll1->add_override(override2);
+
+ results = coll1->search("really amazing shoes", {"name", "category", "brand"}, "",
+ {}, sort_fields, {0, 0, 0}, 1).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}