Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support both include/exclude + filter by override.
Kishore Nallan5483ae46310d
diff --git a/include/collection.h b/include/collection.hindex dc34d89a..ed2fadc2 100644--- a/include/collection.h+++ b/include/collection.h@@ -181,6 +181,13 @@ private:std::vector<field>& del_fields,std::string& fallback_field_type);+ void process_filter_overrides(std::vector<const override_t*>& filter_overrides,+ std::vector<std::string>& q_include_tokens,+ token_ordering token_order,+ std::vector<filter>& filters,+ std::vector<std::pair<uint32_t, uint32_t>>& included_ids,+ std::vector<uint32_t>& excluded_ids) const;+public:enum {MAX_ARRAY_MATCHES = 5};diff --git a/include/index.h b/include/index.hindex 672b4575..606bfbf2 100644--- a/include/index.h+++ b/include/index.h@@ -1032,7 +1032,8 @@ public:void process_filter_overrides(const std::vector<const override_t*>& filter_overrides,std::vector<std::string>& query_tokens,token_ordering token_order,- std::vector<filter>& filters) const;+ std::vector<filter>& filters,+ std::vector<const override_t*>& matched_dynamic_overrides) const;void compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,diff --git a/src/collection.cpp b/src/collection.cppindex 56436e9e..6e5d4cb4 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -454,7 +454,6 @@ void Collection::curate_results(string& actual_query, bool enable_overrides, boo// ID-based overrides are applied first as they take precedence over filter-based overridesif(!override.filter_by.empty()) {filter_overrides.push_back(&override);- continue;}if ((override.rule.match == override_t::MATCH_EXACT && override.rule.query == query) ||@@ -482,7 +481,8 @@ void Collection::curate_results(string& actual_query, bool enable_overrides, boo}}- if(override.remove_matched_tokens) {+ if(override.remove_matched_tokens && override.filter_by.empty()) {+ // don't prematurely remove tokens from query because dynamic filtering will require themStringUtils::replace_all(query, override.rule.query, "");StringUtils::trim(query);if(query.empty()) {@@ -1078,7 +1078,10 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sfield_locale, pre_segmented_query);// process filter overrides first, before synonyms (order is important)- index->process_filter_overrides(filter_overrides, q_include_tokens, token_order, filters);++ // included_ids, excluded_ids+ process_filter_overrides(filter_overrides, q_include_tokens, token_order, filters,+ included_ids, excluded_ids);for(size_t i = 0; i < q_include_tokens.size(); i++) {auto& q_include_token = q_include_tokens[i];@@ -1644,6 +1647,43 @@ void Collection::process_highlight_fields(const std::vector<std::string>& search}}+void Collection::process_filter_overrides(std::vector<const override_t*>& filter_overrides,+ std::vector<std::string>& q_include_tokens,+ token_ordering token_order,+ std::vector<filter>& filters,+ std::vector<std::pair<uint32_t, uint32_t>>& included_ids,+ std::vector<uint32_t>& excluded_ids) const {++ std::vector<const override_t*> matched_dynamic_overrides;+ index->process_filter_overrides(filter_overrides, q_include_tokens, token_order,+ filters, matched_dynamic_overrides);++ // we will check the dynamic overrides to see if they also have include/exclude+ std::set<uint32_t> excluded_set;++ for(auto matched_dynamic_override: matched_dynamic_overrides) {+ for(const auto& hit: matched_dynamic_override->drop_hits) {+ Option<uint32_t> seq_id_op = doc_id_to_seq_id(hit.doc_id);+ if(seq_id_op.ok()) {+ excluded_ids.push_back(seq_id_op.get());+ excluded_set.insert(seq_id_op.get());+ }+ }++ for(const auto& hit: matched_dynamic_override->add_hits) {+ Option<uint32_t> seq_id_op = doc_id_to_seq_id(hit.doc_id);+ if(!seq_id_op.ok()) {+ continue;+ }+ uint32_t seq_id = seq_id_op.get();+ bool excluded = (excluded_set.count(seq_id) != 0);+ if(!excluded) {+ included_ids.emplace_back(seq_id, hit.position);+ }+ }+ }+}+void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,std::vector<std::vector<std::string>>& q_exclude_tokens,std::vector<std::vector<std::string>>& q_phrases,diff --git a/src/index.cpp b/src/index.cppindex 838c82ff..9801dced 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2023,7 +2023,8 @@ bool Index::resolve_override(const std::vector<std::string>& rule_tokens, constvoid Index::process_filter_overrides(const std::vector<const override_t*>& filter_overrides,std::vector<std::string>& query_tokens,token_ordering token_order,- std::vector<filter>& filters) const {+ std::vector<filter>& filters,+ std::vector<const override_t*>& matched_dynamic_overrides) const {std::shared_lock lock(mutex);size_t orig_filters_size = filters.size();@@ -2067,6 +2068,9 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filteOption<bool> filter_parse_op = filter::parse_filter_query(filter_by_clause, search_schema, store, "",filters);if(filter_parse_op.ok()) {+ // have to ensure that dropped hits take precedence over added hits+ matched_dynamic_overrides.push_back(override);+if(override->remove_matched_tokens) {std::vector<std::string>& tokens = query_tokens;remove_matched_tokens(tokens, absorbed_tokens);diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex a4655550..c3401ab2 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -713,6 +713,74 @@ TEST_F(CollectionOverrideTest, SimpleOverrideStopProcessing) {ASSERT_TRUE(override_include_test.stop_processing);}+TEST_F(CollectionOverrideTest, IncludeOverrideWithFilterBy) {+ Collection *coll1;++ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("price", field_types::FLOAT, true),+ field("points", field_types::INT32, false)};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Amazing Shoes";+ doc1["price"] = 399.99;+ doc1["points"] = 30;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "Fast Shoes";+ doc2["price"] = 49.99;+ doc2["points"] = 5;++ nlohmann::json doc3;+ doc3["id"] = "2";+ doc3["name"] = "Comfortable Shoes";+ doc3["price"] = 199.99;+ doc3["points"] = 1;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());++ std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };++ nlohmann::json override_json_include = {+ {"id", "include-rule-1"},+ {+ "rule", {+ {"query", "shoes"},+ {"match", override_t::MATCH_EXACT}+ }+ },+ {"filter_curated_hits", false},+ {"stop_processing", false},+ {"remove_matched_tokens", false},+ {"filter_by", "price: >55"}+ };++ override_json_include["includes"] = nlohmann::json::array();+ override_json_include["includes"][0] = nlohmann::json::object();+ override_json_include["includes"][0]["id"] = "2";+ override_json_include["includes"][0]["position"] = 1;++ override_t override_include1;+ auto op = override_t::parse(override_json_include, "include-rule-1", override_include1);+ ASSERT_TRUE(op.ok());+ coll1->add_override(override_include1);++ auto results = coll1->search("shoes", {"name"}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());+}+TEST_F(CollectionOverrideTest, PinnedAndHiddenHits) {auto pinned_hits = "13:1,4:2";@@ -1232,6 +1300,27 @@ TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {ASSERT_TRUE(op.ok());coll1->add_override(override);+ override_json = {+ {"id", "dynamic-brand-filter"},+ {+ "rule", {+ {"query", "{brand}"},+ {"match", override_t::MATCH_EXACT}+ }+ },+ {"remove_matched_tokens", true},+ {"filter_by", "brand: {brand}"}+ };++ override_json["includes"] = nlohmann::json::array();+ override_json["includes"][0] = nlohmann::json::object();+ override_json["includes"][0]["id"] = "0";+ override_json["includes"][0]["position"] = 1;++ op = override_t::parse(override_json, "dynamic-brand-filter", override);+ ASSERT_TRUE(op.ok());+ coll1->add_override(override);+results = coll1->search("shoes", {"name", "category", "brand"}, "",{}, sort_fields, {2, 2, 2}, 10).get();@@ -1254,6 +1343,14 @@ TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {ASSERT_EQ(1, results["hits"].size());ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ // dynamic brand filter + explicit ID include+ results = coll1->search("adidas", {"name", "category", "brand"}, "",+ {}, sort_fields, {2, 2, 2}, 10).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());+// with bad overridenlohmann::json override_json_bad1 = {