Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Support both include/exclude + filter by override.

Kishore Nallan5483ae46310d
diff --git a/include/collection.h b/include/collection.h
index dc34d89a..ed2fadc2 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -181,6 +181,13 @@ private:
std::vector<field>& del_fields,
std::string& fallback_field_type);
+ void process_filter_overrides(std::vector<const override_t*>& filter_overrides,
+ std::vector<std::string>& q_include_tokens,
+ token_ordering token_order,
+ std::vector<filter>& filters,
+ std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
+ std::vector<uint32_t>& excluded_ids) const;
+
public:
enum {MAX_ARRAY_MATCHES = 5};
diff --git a/include/index.h b/include/index.h
index 672b4575..606bfbf2 100644
--- a/include/index.h
+++ b/include/index.h
@@ -1032,7 +1032,8 @@ public:
void process_filter_overrides(const std::vector<const override_t*>& filter_overrides,
std::vector<std::string>& query_tokens,
token_ordering token_order,
- std::vector<filter>& filters) const;
+ std::vector<filter>& filters,
+ std::vector<const override_t*>& matched_dynamic_overrides) const;
void compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
diff --git a/src/collection.cpp b/src/collection.cpp
index 56436e9e..6e5d4cb4 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -454,7 +454,6 @@ void Collection::curate_results(string& actual_query, bool enable_overrides, boo
// ID-based overrides are applied first as they take precedence over filter-based overrides
if(!override.filter_by.empty()) {
filter_overrides.push_back(&override);
- continue;
}
if ((override.rule.match == override_t::MATCH_EXACT && override.rule.query == query) ||
@@ -482,7 +481,8 @@ void Collection::curate_results(string& actual_query, bool enable_overrides, boo
}
}
- if(override.remove_matched_tokens) {
+ if(override.remove_matched_tokens && override.filter_by.empty()) {
+ // don't prematurely remove tokens from query because dynamic filtering will require them
StringUtils::replace_all(query, override.rule.query, "");
StringUtils::trim(query);
if(query.empty()) {
@@ -1078,7 +1078,10 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
field_locale, pre_segmented_query);
// process filter overrides first, before synonyms (order is important)
- index->process_filter_overrides(filter_overrides, q_include_tokens, token_order, filters);
+
+ // included_ids, excluded_ids
+ process_filter_overrides(filter_overrides, q_include_tokens, token_order, filters,
+ included_ids, excluded_ids);
for(size_t i = 0; i < q_include_tokens.size(); i++) {
auto& q_include_token = q_include_tokens[i];
@@ -1644,6 +1647,43 @@ void Collection::process_highlight_fields(const std::vector<std::string>& search
}
}
+void Collection::process_filter_overrides(std::vector<const override_t*>& filter_overrides,
+ std::vector<std::string>& q_include_tokens,
+ token_ordering token_order,
+ std::vector<filter>& filters,
+ std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
+ std::vector<uint32_t>& excluded_ids) const {
+
+ std::vector<const override_t*> matched_dynamic_overrides;
+ index->process_filter_overrides(filter_overrides, q_include_tokens, token_order,
+ filters, matched_dynamic_overrides);
+
+ // we will check the dynamic overrides to see if they also have include/exclude
+ std::set<uint32_t> excluded_set;
+
+ for(auto matched_dynamic_override: matched_dynamic_overrides) {
+ for(const auto& hit: matched_dynamic_override->drop_hits) {
+ Option<uint32_t> seq_id_op = doc_id_to_seq_id(hit.doc_id);
+ if(seq_id_op.ok()) {
+ excluded_ids.push_back(seq_id_op.get());
+ excluded_set.insert(seq_id_op.get());
+ }
+ }
+
+ for(const auto& hit: matched_dynamic_override->add_hits) {
+ Option<uint32_t> seq_id_op = doc_id_to_seq_id(hit.doc_id);
+ if(!seq_id_op.ok()) {
+ continue;
+ }
+ uint32_t seq_id = seq_id_op.get();
+ bool excluded = (excluded_set.count(seq_id) != 0);
+ if(!excluded) {
+ included_ids.emplace_back(seq_id, hit.position);
+ }
+ }
+ }
+}
+
void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
std::vector<std::vector<std::string>>& q_exclude_tokens,
std::vector<std::vector<std::string>>& q_phrases,
diff --git a/src/index.cpp b/src/index.cpp
index 838c82ff..9801dced 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2023,7 +2023,8 @@ bool Index::resolve_override(const std::vector<std::string>& rule_tokens, const
void Index::process_filter_overrides(const std::vector<const override_t*>& filter_overrides,
std::vector<std::string>& query_tokens,
token_ordering token_order,
- std::vector<filter>& filters) const {
+ std::vector<filter>& filters,
+ std::vector<const override_t*>& matched_dynamic_overrides) const {
std::shared_lock lock(mutex);
size_t orig_filters_size = filters.size();
@@ -2067,6 +2068,9 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filte
Option<bool> filter_parse_op = filter::parse_filter_query(filter_by_clause, search_schema, store, "",
filters);
if(filter_parse_op.ok()) {
+ // have to ensure that dropped hits take precedence over added hits
+ matched_dynamic_overrides.push_back(override);
+
if(override->remove_matched_tokens) {
std::vector<std::string>& tokens = query_tokens;
remove_matched_tokens(tokens, absorbed_tokens);
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index a4655550..c3401ab2 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -713,6 +713,74 @@ TEST_F(CollectionOverrideTest, SimpleOverrideStopProcessing) {
ASSERT_TRUE(override_include_test.stop_processing);
}
+TEST_F(CollectionOverrideTest, IncludeOverrideWithFilterBy) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("price", field_types::FLOAT, true),
+ field("points", field_types::INT32, false)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Amazing Shoes";
+ doc1["price"] = 399.99;
+ doc1["points"] = 30;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "Fast Shoes";
+ doc2["price"] = 49.99;
+ doc2["points"] = 5;
+
+ nlohmann::json doc3;
+ doc3["id"] = "2";
+ doc3["name"] = "Comfortable Shoes";
+ doc3["price"] = 199.99;
+ doc3["points"] = 1;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());
+
+ std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
+
+ nlohmann::json override_json_include = {
+ {"id", "include-rule-1"},
+ {
+ "rule", {
+ {"query", "shoes"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ },
+ {"filter_curated_hits", false},
+ {"stop_processing", false},
+ {"remove_matched_tokens", false},
+ {"filter_by", "price: >55"}
+ };
+
+ override_json_include["includes"] = nlohmann::json::array();
+ override_json_include["includes"][0] = nlohmann::json::object();
+ override_json_include["includes"][0]["id"] = "2";
+ override_json_include["includes"][0]["position"] = 1;
+
+ override_t override_include1;
+ auto op = override_t::parse(override_json_include, "include-rule-1", override_include1);
+ ASSERT_TRUE(op.ok());
+ coll1->add_override(override_include1);
+
+ auto results = coll1->search("shoes", {"name"}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+}
+
TEST_F(CollectionOverrideTest, PinnedAndHiddenHits) {
auto pinned_hits = "13:1,4:2";
@@ -1232,6 +1300,27 @@ TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {
ASSERT_TRUE(op.ok());
coll1->add_override(override);
+ override_json = {
+ {"id", "dynamic-brand-filter"},
+ {
+ "rule", {
+ {"query", "{brand}"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ },
+ {"remove_matched_tokens", true},
+ {"filter_by", "brand: {brand}"}
+ };
+
+ override_json["includes"] = nlohmann::json::array();
+ override_json["includes"][0] = nlohmann::json::object();
+ override_json["includes"][0]["id"] = "0";
+ override_json["includes"][0]["position"] = 1;
+
+ op = override_t::parse(override_json, "dynamic-brand-filter", override);
+ ASSERT_TRUE(op.ok());
+ coll1->add_override(override);
+
results = coll1->search("shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10).get();
@@ -1254,6 +1343,14 @@ TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ // dynamic brand filter + explicit ID include
+ results = coll1->search("adidas", {"name", "category", "brand"}, "",
+ {}, sort_fields, {2, 2, 2}, 10).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+
// with bad override
nlohmann::json override_json_bad1 = {