Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Enable filtering via overrides.
Kishore Nallanc6fe1369b957
diff --git a/include/collection.h b/include/collection.hindex f6e5a244..b1697f2b 100644--- a/include/collection.h+++ b/include/collection.h@@ -19,151 +19,6 @@#include <option.h>#include "tokenizer.h"--struct override_t {- static const std::string MATCH_EXACT;- static const std::string MATCH_CONTAINS;-- struct rule_t {- std::string query;- std::string match;- };-- struct add_hit_t {- std::string doc_id;- uint32_t position;- };-- struct drop_hit_t {- std::string doc_id;- };-- std::string id;- rule_t rule;- std::vector<add_hit_t> add_hits;- std::vector<drop_hit_t> drop_hits;-- override_t() {}-- static Option<bool> parse(const nlohmann::json& override_json, const std::string& id, override_t& override) {- if(!override_json.is_object()) {- return Option<bool>(400, "Bad JSON.");- }-- if(override_json.count("rule") == 0 || !override_json["rule"].is_object()) {- return Option<bool>(400, "Missing `rule` definition.");- }-- if(override_json["rule"].count("query") == 0 || override_json["rule"].count("match") == 0) {- return Option<bool>(400, "The `rule` definition must contain a `query` or `match`.");- }-- if(override_json.count("includes") == 0 && override_json.count("excludes") == 0) {- return Option<bool>(400, "Must contain either `includes` or `excludes`.");- }-- if(override_json.count("includes") != 0) {- if(!override_json["includes"].is_array()) {- return Option<bool>(400, "The `includes` value must be an array.");- }-- for(const auto & include_obj: override_json["includes"]) {- if(!include_obj.is_object()) {- return Option<bool>(400, "The `includes` value must be an array of objects.");- }-- if(include_obj.count("id") == 0 || include_obj.count("position") == 0) {- return Option<bool>(400, "Inclusion definition must define both `id` and `position` keys.");- }-- if(!include_obj["id"].is_string()) {- return Option<bool>(400, "Inclusion `id` must be a string.");- }-- if(!include_obj["position"].is_number_integer()) {- return Option<bool>(400, "Inclusion `position` must be an integer.");- }- }- }-- if(override_json.count("excludes") != 0) {- if(!override_json["excludes"].is_array()) {- return Option<bool>(400, "The `excludes` value must be an array.");- }-- for(const auto & exclude_obj: override_json["excludes"]) {- if(!exclude_obj.is_object()) {- return Option<bool>(400, "The `excludes` value must be an array of objects.");- }-- if(exclude_obj.count("id") == 0) {- return Option<bool>(400, "Exclusion definition must define an `id`.");- }-- if(!exclude_obj["id"].is_string()) {- return Option<bool>(400, "Exclusion `id` must be a string.");- }- }-- }-- if(!id.empty()) {- override.id = id;- } else if(override_json.count("id") != 0) {- override.id = override_json["id"].get<std::string>();- } else {- return Option<bool>(400, "Override `id` not provided.");- }-- override.rule.query = override_json["rule"]["query"].get<std::string>();- override.rule.match = override_json["rule"]["match"].get<std::string>();-- if (override_json.count("includes") != 0) {- for(const auto & include: override_json["includes"]) {- add_hit_t add_hit;- add_hit.doc_id = include["id"].get<std::string>();- add_hit.position = include["position"].get<uint32_t>();- override.add_hits.push_back(add_hit);- }- }-- if (override_json.count("excludes") != 0) {- for(const auto & exclude: override_json["excludes"]) {- drop_hit_t drop_hit;- drop_hit.doc_id = exclude["id"].get<std::string>();- override.drop_hits.push_back(drop_hit);- }- }-- return Option<bool>(true);- }-- nlohmann::json to_json() const {- nlohmann::json override;- override["id"] = id;- override["rule"]["query"] = rule.query;- override["rule"]["match"] = rule.match;-- override["includes"] = nlohmann::json::array();-- for(const auto & add_hit: add_hits) {- nlohmann::json include;- include["id"] = add_hit.doc_id;- include["position"] = add_hit.position;- override["includes"].push_back(include);- }-- override["excludes"] = nlohmann::json::array();- for(const auto & drop_hit: drop_hits) {- nlohmann::json exclude;- exclude["id"] = drop_hit.doc_id;- override["excludes"].push_back(exclude);- }-- return override;- }-};-struct doc_seq_id_t {uint32_t seq_id;bool is_new;@@ -348,12 +203,12 @@ private:void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);- void curate_results(std::string query,+ void curate_results(string& actual_query, bool enable_overrides, bool already_segmented,const std::map<size_t, std::vector<std::string>>& pinned_hits,const std::vector<std::string>& hidden_hits,std::map<size_t, std::vector<uint32_t>>& include_ids,- std::vector<uint32_t> & excluded_ids,- bool enable_overrides) const;+ std::vector<uint32_t>& excluded_ids, std::vector<const override_t*>& dynamic_overrides,+ std::vector<filter>& filters) const;Option<bool> check_and_update_schema(nlohmann::json& document, const DIRTY_VALUES& dirty_values);diff --git a/include/index.h b/include/index.hindex 97c29dec..3c0042e4 100644--- a/include/index.h+++ b/include/index.h@@ -47,6 +47,181 @@ struct query_tokens_t {std::vector<std::vector<std::string>> q_synonyms;};+struct override_t {+ static const std::string MATCH_EXACT;+ static const std::string MATCH_CONTAINS;++ struct rule_t {+ std::string query;+ std::string match;+ };++ struct add_hit_t {+ std::string doc_id;+ uint32_t position;+ };++ struct drop_hit_t {+ std::string doc_id;+ };++ std::string id;++ rule_t rule;+ std::vector<add_hit_t> add_hits;+ std::vector<drop_hit_t> drop_hits;++ std::vector<std::string> dynamic_filters;+ bool mutate_query_string = false;++ override_t() {}++ static Option<bool> parse(const nlohmann::json& override_json, const std::string& id, override_t& override) {+ if(!override_json.is_object()) {+ return Option<bool>(400, "Bad JSON.");+ }++ if(override_json.count("rule") == 0 || !override_json["rule"].is_object()) {+ return Option<bool>(400, "Missing `rule` definition.");+ }++ if(override_json["rule"].count("query") == 0 || override_json["rule"].count("match") == 0) {+ return Option<bool>(400, "The `rule` definition must contain a `query` and `match`.");+ }++ if(override_json.count("includes") == 0 && override_json.count("excludes") == 0 &&+ override_json.count("dynamic_filters") == 0) {+ return Option<bool>(400, "Must contain one of:`includes`, `excludes`, `dynamic_filters`.");+ }++ if(override_json.count("includes") != 0) {+ if(!override_json["includes"].is_array()) {+ return Option<bool>(400, "The `includes` value must be an array.");+ }++ for(const auto & include_obj: override_json["includes"]) {+ if(!include_obj.is_object()) {+ return Option<bool>(400, "The `includes` value must be an array of objects.");+ }++ if(include_obj.count("id") == 0 || include_obj.count("position") == 0) {+ return Option<bool>(400, "Inclusion definition must define both `id` and `position` keys.");+ }++ if(!include_obj["id"].is_string()) {+ return Option<bool>(400, "Inclusion `id` must be a string.");+ }++ if(!include_obj["position"].is_number_integer()) {+ return Option<bool>(400, "Inclusion `position` must be an integer.");+ }+ }+ }++ if(override_json.count("excludes") != 0) {+ if(!override_json["excludes"].is_array()) {+ return Option<bool>(400, "The `excludes` value must be an array.");+ }++ for(const auto & exclude_obj: override_json["excludes"]) {+ if(!exclude_obj.is_object()) {+ return Option<bool>(400, "The `excludes` value must be an array of objects.");+ }++ if(exclude_obj.count("id") == 0) {+ return Option<bool>(400, "Exclusion definition must define an `id`.");+ }++ if(!exclude_obj["id"].is_string()) {+ return Option<bool>(400, "Exclusion `id` must be a string.");+ }+ }++ }++ if(override_json.count("dynamic_filters") != 0) {+ if(!override_json["dynamic_filters"].is_string()) {+ return Option<bool>(400, "The `dynamic_filters` must be "+ "a comma separated string of filter field names.");+ }++ if(override_json["dynamic_filters"].get<std::string>().empty()) {+ return Option<bool>(400, "The `dynamic_filters` must be "+ "a comma separated string of filter field names.");+ }+ }++ if(override_json.count("mutate_query_string") != 0) {+ if (!override_json["mutate_query_string"].is_boolean()) {+ return Option<bool>(400, "The `mutate_query_string` must be a boolean.");+ }++ override.mutate_query_string = override_json["mutate_query_string"].get<bool>();+ }++ if(!id.empty()) {+ override.id = id;+ } else if(override_json.count("id") != 0) {+ override.id = override_json["id"].get<std::string>();+ } else {+ return Option<bool>(400, "Override `id` not provided.");+ }++ override.rule.query = override_json["rule"]["query"].get<std::string>();+ override.rule.match = override_json["rule"]["match"].get<std::string>();++ if (override_json.count("includes") != 0) {+ for(const auto & include: override_json["includes"]) {+ add_hit_t add_hit;+ add_hit.doc_id = include["id"].get<std::string>();+ add_hit.position = include["position"].get<uint32_t>();+ override.add_hits.push_back(add_hit);+ }+ }++ if (override_json.count("excludes") != 0) {+ for(const auto & exclude: override_json["excludes"]) {+ drop_hit_t drop_hit;+ drop_hit.doc_id = exclude["id"].get<std::string>();+ override.drop_hits.push_back(drop_hit);+ }+ }++ if (override_json.count("dynamic_filters") != 0) {+ std::vector<std::string> filter_fields;+ StringUtils::split(override_json["dynamic_filters"].get<std::string>(), filter_fields, ",");+ override.dynamic_filters = filter_fields;+ }++ return Option<bool>(true);+ }++ nlohmann::json to_json() const {+ nlohmann::json override;+ override["id"] = id;+ override["rule"]["query"] = rule.query;+ override["rule"]["match"] = rule.match;++ override["includes"] = nlohmann::json::array();++ for(const auto & add_hit: add_hits) {+ nlohmann::json include;+ include["id"] = add_hit.doc_id;+ include["position"] = add_hit.position;+ override["includes"].push_back(include);+ }++ override["excludes"] = nlohmann::json::array();+ for(const auto & drop_hit: drop_hits) {+ nlohmann::json exclude;+ exclude["id"] = drop_hit.doc_id;+ override["excludes"].push_back(exclude);+ }++ return override;+ }+};+struct search_args {std::vector<query_tokens_t> field_query_tokens;std::vector<search_field_t> search_fields;@@ -71,6 +246,7 @@ struct search_args {size_t all_result_ids_len;bool exhaustive_search;size_t concurrency;+ const std::vector<const override_t*>& dynamic_overrides;spp::sparse_hash_set<uint64_t> groups_processed;std::vector<std::vector<art_leaf*>> searched_queries;@@ -91,7 +267,8 @@ struct search_args {const std::string& default_sorting_field,bool prioritize_exact_match,bool exhaustive_search,- size_t concurrency):+ size_t concurrency,+ const std::vector<const override_t*>& dynamic_overrides):field_query_tokens(field_query_tokens),search_fields(search_fields), filters(filters), facets(facets),included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),@@ -100,7 +277,8 @@ struct search_args {drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),prioritize_exact_match(prioritize_exact_match), all_result_ids_len(0),- exhaustive_search(exhaustive_search), concurrency(concurrency) {+ exhaustive_search(exhaustive_search), concurrency(concurrency),+ dynamic_overrides(dynamic_overrides) {const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatorytopster = new Topster(topster_size, group_limit);@@ -218,6 +396,12 @@ private:size_t group_limit, const std::vector<std::string>& group_by_fields,const uint32_t* result_ids, size_t results_size) const;+ void process_dynamic_overrides(const std::vector<const override_t*>& dynamic_overrides,+ std::vector<query_tokens_t>& field_query_tokens,+ const token_ordering token_order,+ uint32_t** filter_ids,+ uint32_t& filter_ids_length) const;+void search_field(const uint8_t & field_id,std::vector<token_t>& query_tokens,std::vector<token_t>& search_tokens,@@ -374,7 +558,7 @@ public:void run_search(search_args* search_params);- void search(const std::vector<query_tokens_t>& field_query_tokens,+ void search(std::vector<query_tokens_t>& field_query_tokens,const std::vector<search_field_t> & search_fields,const std::vector<filter> & filters, std::vector<facet> & facets,facet_query_t & facet_query,@@ -392,6 +576,7 @@ public:const size_t typo_tokens_threshold,const size_t group_limit,const std::vector<std::string>& group_by_fields,+ const std::vector<const override_t*>& dynamic_overrides,const std::string& default_sorting_field,bool prioritize_exact_match,bool exhaustive_search,@@ -434,5 +619,23 @@ public:const std::string& fallback_field_type,const DIRTY_VALUES& dirty_values);+ void check_for_overrides(const token_ordering& token_order, const string& field_name, bool mutate_query_string,+ uint32_t*& field_override_ids,+ size_t& field_override_ids_len, std::vector<std::string>& tokens) const;++ void search_wildcard(const std::vector<std::string>& qtokens, const std::vector<filter>& filters,+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ const std::vector<sort_by>& sort_fields_std, Topster* topster, Topster* curated_topster,+ spp::sparse_hash_set<uint64_t>& groups_processed,+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,+ const std::vector<std::string>& group_by_fields,+ const std::set<uint32_t>& curated_ids, const std::vector<uint32_t>& curated_ids_sorted,+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size, const uint8_t field_id,+ const string& field, uint32_t*& all_result_ids, size_t& all_result_ids_len,+ const uint32_t* filter_ids, uint32_t filter_ids_length) const;++ void curate_filtered_ids(const std::vector<filter>& filters, const std::set<uint32_t>& curated_ids,+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size, uint32_t*& filter_ids,+ uint32_t& filter_ids_length, const std::vector<uint32_t>& curated_ids_sorted) const;};diff --git a/src/collection.cpp b/src/collection.cppindex 1e8268cf..56bb731c 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -373,13 +373,13 @@ void Collection::prune_document(nlohmann::json &document, const spp::sparse_hash}}-void Collection::curate_results(std::string query,+void Collection::curate_results(string& actual_query, bool enable_overrides, bool already_segmented,const std::map<size_t, std::vector<std::string>>& pinned_hits,const std::vector<std::string>& hidden_hits,std::map<size_t, std::vector<uint32_t>>& include_ids,- std::vector<uint32_t> & excluded_ids,- bool enable_overrides) const {- StringUtils::tolowercase(query);+ std::vector<uint32_t>& excluded_ids, std::vector<const override_t*>& dynamic_overrides,+ std::vector<filter>& filters) const {+std::set<uint32_t> excluded_set;// If pinned or hidden hits are provided, they take precedence over overrides@@ -395,9 +395,19 @@ void Collection::curate_results(std::string query,}}- if(enable_overrides) {- for(const auto & override_kv: overrides) {- const auto & override = override_kv.second;+ std::string query = actual_query;++ if(enable_overrides && !overrides.empty()) {+ StringUtils::tolowercase(query);++ for(const auto& override_kv: overrides) {+ const auto& override = override_kv.second;++ // static overrides are applied first as they precedence over dynamic overrides+ if(!override.dynamic_filters.empty() && override.rule.query == ".*") {+ dynamic_overrides.push_back(&override);+ continue;+ }if( (override.rule.match == override_t::MATCH_EXACT && override.rule.query == query) ||(override.rule.match == override_t::MATCH_CONTAINS && query.find(override.rule.query) != std::string::npos) ) {@@ -422,6 +432,21 @@ void Collection::curate_results(std::string query,include_ids[hit.position].push_back(seq_id);}}++ for(const auto& filter_str: override.dynamic_filters) {+ auto op = parse_filter_query(filter_str, filters);+ if(op.ok() && override.mutate_query_string) {+ actual_query.replace(+ actual_query.find(override.rule.query),+ override.rule.query.size(),+ ""+ );++ if(StringUtils::trim(actual_query).empty()) {+ actual_query = "*";+ }+ }+ }}}}@@ -444,7 +469,7 @@ void Collection::curate_results(std::string query,}}-Option<nlohmann::json> Collection::search(const std::string & query, const std::vector<std::string>& search_fields,+Option<nlohmann::json> Collection::search(const std::string & raw_query, const std::vector<std::string>& search_fields,const std::string & simple_filter_query, const std::vector<std::string>& facet_fields,const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,const size_t per_page, const size_t page,@@ -474,7 +499,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::std::shared_lock lock(mutex);- if(query != "*" && search_fields.empty()) {+ if(raw_query != "*" && search_fields.empty()) {return Option<nlohmann::json>(400, "No search fields specified for the query.");}@@ -502,57 +527,6 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}}- std::vector<uint32_t> excluded_ids;- std::map<size_t, std::vector<uint32_t>> include_ids; // position => list of IDs- std::map<size_t, std::vector<std::string>> pinned_hits;-- Option<bool> pinned_hits_op = parse_pinned_hits(pinned_hits_str, pinned_hits);-- if(!pinned_hits_op.ok()) {- return Option<nlohmann::json>(400, pinned_hits_op.error());- }-- std::vector<std::string> hidden_hits;- StringUtils::split(hidden_hits_str, hidden_hits, ",");-- curate_results(query, pinned_hits, hidden_hits, include_ids, excluded_ids, enable_overrides);-- /*for(auto& kv: include_ids) {- LOG(INFO) << "key: " << kv.first;- for(auto val: kv.second) {- LOG(INFO) << val;- }- }-- LOG(INFO) << "Excludes:";-- for(auto id: excluded_ids) {- LOG(INFO) << id;- }-- LOG(INFO) << "include_ids size: " << include_ids.size();- for(auto& group: include_ids) {- for(uint32_t& seq_id: group.second) {- LOG(INFO) << "seq_id: " << seq_id;- }-- LOG(INFO) << "----";- }- */-- std::map<size_t, std::map<size_t, uint32_t>> included_ids;-- for(const auto& pos_ids: include_ids) {- size_t outer_pos = pos_ids.first;- size_t ids_per_pos = std::max(size_t(1), group_limit);-- for(size_t inner_pos = 0; inner_pos < std::min(ids_per_pos, pos_ids.second.size()); inner_pos++) {- auto seq_id = pos_ids.second[inner_pos];- included_ids[outer_pos][inner_pos] = seq_id;- //LOG(INFO) << "Adding seq_id " << seq_id << " to index_id " << index_id;- }- }-// process weights for search fieldsstd::vector<search_field_t> weighted_search_fields;size_t max_weight = 20;@@ -833,7 +807,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::size_t max_hits = 250;// ensure that `max_hits` never exceeds number of documents in collection- if(search_fields.size() <= 1 || query == "*") {+ if(search_fields.size() <= 1 || raw_query == "*") {max_hits = std::min(std::max((page * per_page), max_hits), get_num_documents());} else {max_hits = std::min(std::max((page * per_page), max_hits), get_num_documents());@@ -853,6 +827,60 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::size_t total_found = 0;spp::sparse_hash_set<uint64_t> groups_processed; // used to calculate total_found for grouped query+ std::vector<uint32_t> excluded_ids;+ std::map<size_t, std::vector<uint32_t>> include_ids; // position => list of IDs+ std::map<size_t, std::vector<std::string>> pinned_hits;++ Option<bool> pinned_hits_op = parse_pinned_hits(pinned_hits_str, pinned_hits);++ if(!pinned_hits_op.ok()) {+ return Option<nlohmann::json>(400, pinned_hits_op.error());+ }++ std::vector<std::string> hidden_hits;+ StringUtils::split(hidden_hits_str, hidden_hits, ",");++ std::vector<const override_t*> dynamic_overrides;+ std::string query = raw_query;+ curate_results(query, enable_overrides, pre_segmented_query, pinned_hits, hidden_hits,+ include_ids, excluded_ids, dynamic_overrides, filters);++ /*for(auto& kv: include_ids) {+ LOG(INFO) << "key: " << kv.first;+ for(auto val: kv.second) {+ LOG(INFO) << val;+ }+ }++ LOG(INFO) << "Excludes:";++ for(auto id: excluded_ids) {+ LOG(INFO) << id;+ }++ LOG(INFO) << "include_ids size: " << include_ids.size();+ for(auto& group: include_ids) {+ for(uint32_t& seq_id: group.second) {+ LOG(INFO) << "seq_id: " << seq_id;+ }++ LOG(INFO) << "----";+ }+ */++ std::map<size_t, std::map<size_t, uint32_t>> included_ids;++ for(const auto& pos_ids: include_ids) {+ size_t outer_pos = pos_ids.first;+ size_t ids_per_pos = std::max(size_t(1), group_limit);++ for(size_t inner_pos = 0; inner_pos < std::min(ids_per_pos, pos_ids.second.size()); inner_pos++) {+ auto seq_id = pos_ids.second[inner_pos];+ included_ids[outer_pos][inner_pos] = seq_id;+ //LOG(INFO) << "Adding seq_id " << seq_id << " to index_id " << index_id;+ }+ }+//LOG(INFO) << "Num indices used for querying: " << indices.size();std::vector<query_tokens_t> field_query_tokens;@@ -888,8 +916,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::per_page, page, token_order, prefixes,drop_tokens_threshold, typo_tokens_threshold,group_by_fields, group_limit, default_sorting_field, prioritize_exact_match,- exhaustive_search,- 4);+ exhaustive_search, 4, dynamic_overrides);index->run_search(search_params);@@ -2337,6 +2364,10 @@ Option<bool> Collection::parse_pinned_hits(const std::string& pinned_hits_str,void Collection::synonym_reduction(const std::vector<std::string>& tokens,std::vector<std::vector<std::string>>& results) const {+ if(synonym_definitions.empty()) {+ return;+ }+std::set<uint64_t> processed_syn_hashes;synonym_reduction_internal(tokens, tokens.size(), 0, processed_syn_hashes, results);}diff --git a/src/index.cpp b/src/index.cppindex 96d32488..7148047f 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1004,35 +1004,40 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,}std::vector<uint32_t> result_id_vecs[concurrency];-Topster* topsters[concurrency];- for(size_t i = 0; i < concurrency; i++) {- topsters[i] = new Topster(topster->MAX_SIZE, topster->distinct);- }-std::vector<spp::sparse_hash_set<uint64_t>> groups_processed_vec(concurrency);- posting_t::block_intersector_t(- posting_lists, iter_state, thread_pool, 100- )- .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its, size_t index) {- score_results(sort_fields, searched_queries.size(), field_id, field_is_array,- total_cost, topsters[index], query_suggestion, groups_processed_vec[index],- seq_id, sort_order, field_values, geopoint_indices,- group_limit, group_by_fields, token_bits,- prioritize_exact_match, single_exact_query_token, its);+ if(topster == nullptr) {+ posting_t::block_intersector_t(+ posting_lists, iter_state, thread_pool, 100+ )+ .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its, size_t index) {+ result_id_vecs[index].push_back(seq_id);+ }, concurrency);+ } else {+ for(size_t i = 0; i < concurrency; i++) {+ topsters[i] = new Topster(topster->MAX_SIZE, topster->distinct);+ }- result_id_vecs[index].push_back(seq_id);- }, concurrency);+ posting_t::block_intersector_t(+ posting_lists, iter_state, thread_pool, 100+ )+ .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its, size_t index) {+ score_results(sort_fields, searched_queries.size(), field_id, field_is_array,+ total_cost, topsters[index], query_suggestion, groups_processed_vec[index],+ seq_id, sort_order, field_values, geopoint_indices,+ group_limit, group_by_fields, token_bits,+ prioritize_exact_match, single_exact_query_token, its);++ result_id_vecs[index].push_back(seq_id);+ }, concurrency);+ }delete [] excluded_result_ids;size_t num_result_ids = 0;for(size_t i = 0; i < concurrency; i++) {- aggregate_topster(*topster, topsters[i]);- delete topsters[i];-uint32_t* new_all_result_ids = nullptr;all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &result_id_vecs[i][0],result_id_vecs[i].size(), &new_all_result_ids);@@ -1041,7 +1046,11 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,num_result_ids += result_id_vecs[i].size();- groups_processed.insert(groups_processed_vec[i].begin(), groups_processed_vec[i].end());+ if(topster != nullptr) {+ aggregate_topster(*topster, topsters[i]);+ delete topsters[i];+ groups_processed.insert(groups_processed_vec[i].begin(), groups_processed_vec[i].end());+ }}if(num_result_ids == 0) {@@ -1400,6 +1409,7 @@ void Index::run_search(search_args* search_params) {search_params->raw_result_kvs, search_params->override_result_kvs,search_params->typo_tokens_threshold,search_params->group_limit, search_params->group_by_fields,+ search_params->dynamic_overrides,search_params->default_sorting_field,search_params->prioritize_exact_match,search_params->exhaustive_search,@@ -1421,6 +1431,10 @@ void Index::collate_included_ids(const std::vector<std::string>& q_included_tokestd::vector<art_leaf *> override_query;for(const std::string& token: q_included_tokens) {+ if(token == "*") {+ continue;+ }+const size_t token_len = token.size() + 1;std::vector<art_leaf*> leaves;@@ -1475,7 +1489,140 @@ void Index::concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t,}}-void Index::search(const std::vector<query_tokens_t>& field_query_tokens,+void Index::process_dynamic_overrides(const std::vector<const override_t*>& dynamic_overrides,+ std::vector<query_tokens_t>& field_query_tokens,+ token_ordering token_order,+ uint32_t** filter_ids,+ uint32_t& filter_ids_length) const {++ // compute dynamic filters if any+ for(auto& override: dynamic_overrides) {+ // will do an AND across different fields+ uint32_t* field_override_ids = nullptr;+ size_t field_override_ids_len = 0;++ // check for dynamic filter+ for(auto& field_name: override->dynamic_filters) {+ // check if query matches a filter field+ std::vector<std::string> tokens = field_query_tokens[0].q_include_tokens;+ check_for_overrides(token_order, field_name, override->mutate_query_string,+ field_override_ids, field_override_ids_len, tokens);++ for(size_t i=0; i < field_query_tokens.size(); i++) {+ if(tokens.empty()) {+ field_query_tokens[i].q_include_tokens = {"*"};+ } else {+ field_query_tokens[i].q_include_tokens = tokens;+ }+ }++ for(size_t i = 0; i < field_query_tokens[0].q_synonyms.size(); i++) {+ uint32_t* synonym_override_ids = nullptr;+ size_t synonym_override_ids_len = 0;++ std::vector<std::string> syn_tokens = field_query_tokens[0].q_synonyms[i];+ check_for_overrides(token_order, field_name, override->mutate_query_string,+ synonym_override_ids, synonym_override_ids_len,syn_tokens);++ for(size_t j=0; j < field_query_tokens.size(); j++) {+ if(syn_tokens.empty()) {+ field_query_tokens[j].q_synonyms[i] = {"*"};+ } else {+ field_query_tokens[j].q_synonyms[i] = syn_tokens;+ }+ }++ // result ids of synonyms will be ORed++ uint32_t* syn_field_override_ids = nullptr;+ field_override_ids_len = ArrayUtils::or_scalar(field_override_ids, field_override_ids_len,+ synonym_override_ids,+ synonym_override_ids_len, &syn_field_override_ids);++ delete [] synonym_override_ids;+ delete [] field_override_ids;+ field_override_ids = syn_field_override_ids;+ }+ }++ if(filter_ids_length != 0) {+ uint32_t* filtered_results = nullptr;+ filter_ids_length = ArrayUtils::and_scalar(field_override_ids, field_override_ids_len, *filter_ids,+ filter_ids_length, &filtered_results);+ delete [] *filter_ids;+ delete [] field_override_ids;+ *filter_ids = filtered_results;+ } else {+ *filter_ids = field_override_ids;+ filter_ids_length = field_override_ids_len;+ }+ }+}++void+Index::check_for_overrides(const token_ordering& token_order, const string& field_name, bool mutate_query_string,+ uint32_t*& field_override_ids,+ size_t& field_override_ids_len, std::vector<std::string>& actual_tokens) const {++ std::vector<std::string> local_tokens = actual_tokens;+ std::vector<std::string>& tokens = mutate_query_string ? actual_tokens : local_tokens;++ DO_WINDOWING:+ for(size_t window_len = tokens.size(); window_len > 0; window_len--) {+ for(size_t start_index = 0; start_index+window_len-1 < tokens.size(); start_index++) {+ std::vector<token_t> window_tokens;+ std::set<std::string> window_tokens_set;+ for (size_t i = start_index; i < start_index + window_len; i++) {+ window_tokens.push_back({i, tokens[i]});+ window_tokens_set.emplace(tokens[i]);+ }++ std::vector<token_t> search_tokens = window_tokens;++ std::vector<facet> facets;+ std::vector<std::vector<art_leaf*>> searched_queries;+ Topster* topster = nullptr;+ spp::sparse_hash_set<uint64_t> groups_processed;+ uint32_t* result_ids = nullptr;+ size_t result_ids_len = 0;+ size_t field_num_results = 0;+ std::vector<std::string> group_by_fields;++ size_t num_toks_dropped = 0;+ search_field(0, window_tokens, search_tokens, nullptr, 0, num_toks_dropped, field_name,+ nullptr, 0, {}, facets, {}, 2, searched_queries, topster, groups_processed,+ &result_ids, result_ids_len, field_num_results, 0, group_by_fields,+ false, 4, token_order, false, 0, 1, false);++ if(result_ids_len != 0) {+ if(field_override_ids != nullptr) {+ uint32_t* filtered_results = nullptr;+ field_override_ids_len = ArrayUtils::and_scalar(field_override_ids, field_override_ids_len, result_ids,+ result_ids_len, &filtered_results);+ delete [] result_ids;+ delete [] field_override_ids;+ field_override_ids = filtered_results;+ } else {+ field_override_ids = result_ids;+ field_override_ids_len = result_ids_len;+ }++ // remove window_tokens from `tokens` and repeat search+ std::vector<std::string> new_tokens;+ for(const auto& token: tokens) {+ if(window_tokens_set.count(token) == 0) {+ new_tokens.emplace_back(token);+ }+ }++ tokens = new_tokens;+ goto DO_WINDOWING;+ }+ }+ }+}++void Index::search(std::vector<query_tokens_t>& field_query_tokens,const std::vector<search_field_t>& search_fields,const std::vector<filter>& filters,std::vector<facet>& facets, facet_query_t& facet_query,@@ -1494,6 +1641,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,const size_t typo_tokens_threshold,const size_t group_limit,const std::vector<std::string>& group_by_fields,+ const std::vector<const override_t*>& dynamic_overrides,const std::string& default_sorting_field,bool prioritize_exact_match,const bool exhaustive_search,@@ -1524,6 +1672,8 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,std::vector<uint32_t> curated_ids_sorted(curated_ids.begin(), curated_ids.end());std::sort(curated_ids_sorted.begin(), curated_ids_sorted.end());+ process_dynamic_overrides(dynamic_overrides, field_query_tokens, token_order, &filter_ids, filter_ids_length);+// Order of `fields` are used to sort results//auto begin = std::chrono::high_resolution_clock::now();uint32_t* all_result_ids = nullptr;@@ -1568,70 +1718,14 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);const std::string& field = search_fields[0].name;- // if a filter is not specified, use the seq_ids index to generate the list of all document ids- if(filters.empty()) {- filter_ids_length = seq_ids.getLength();- filter_ids = seq_ids.uncompress();- }-- if(!curated_ids.empty()) {- uint32_t *excluded_result_ids = nullptr;- filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, &curated_ids_sorted[0],- curated_ids_sorted.size(), &excluded_result_ids);- delete [] filter_ids;- filter_ids = excluded_result_ids;- }+ curate_filtered_ids(filters, curated_ids, exclude_token_ids,+ exclude_token_ids_size, filter_ids, filter_ids_length, curated_ids_sorted);- // Exclude document IDs associated with excluded tokens from the result set- if(exclude_token_ids_size != 0) {- uint32_t *excluded_result_ids = nullptr;- filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, exclude_token_ids,- exclude_token_ids_size, &excluded_result_ids);- delete[] filter_ids;- filter_ids = excluded_result_ids;- }--- // FIXME: duplicated- int sort_order[3]; // 1 or -1 based on DESC or ASC respectively- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values;- std::vector<size_t> geopoint_indices;-- for (size_t i = 0; i < sort_fields_std.size(); i++) {- sort_order[i] = 1;- if (sort_fields_std[i].order == sort_field_const::asc) {- sort_order[i] = -1;- }-- if (sort_fields_std[i].name == sort_field_const::text_match) {- field_values[i] = &text_match_sentinel_value;- } else if (sort_fields_std[i].name == sort_field_const::seq_id) {- field_values[i] = &seq_id_sentinel_value;- } else if (sort_index.count(sort_fields_std[i].name) != 0) {- field_values[i] = sort_index.at(sort_fields_std[i].name);-- if (sort_schema.at(sort_fields_std[i].name).is_geopoint()) {- geopoint_indices.push_back(i);- }- }- }-- uint32_t token_bits = 255;- std::vector<posting_list_t::iterator_t> plists;-- for(size_t i = 0; i < filter_ids_length; i++) {- const uint32_t seq_id = filter_ids[i];- score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, false, 0, topster, {},- groups_processed, seq_id, sort_order, field_values,- geopoint_indices, group_limit, group_by_fields, token_bits,- false, false, plists);- }-- collate_included_ids(field_query_tokens[0].q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);-- all_result_ids_len = filter_ids_length;- all_result_ids = filter_ids;- filter_ids = nullptr;+ search_wildcard(field_query_tokens[0].q_include_tokens, filters, included_ids_map, sort_fields_std, topster,+ curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,+ curated_ids, curated_ids_sorted,+ exclude_token_ids, exclude_token_ids_size, field_id, field,+ all_result_ids, all_result_ids_len, filter_ids, filter_ids_length);} else {// In multi-field searches, a record can be matched across different fields, so we use this for aggregationspp::sparse_hash_map<uint64_t, std::vector<KV*>> topster_ids;@@ -1685,18 +1779,37 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,token_order, field_prefix,drop_tokens_threshold, typo_tokens_threshold, exhaustive_search);+ bool syn_wildcard_filter_init_done = false;+// do synonym based searchesfor(const auto& syn_tokens: q_pos_synonyms) {num_tokens_dropped = 0;field_num_results = 0;query_tokens = search_tokens = syn_tokens;- search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,- field_name, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,- field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,- field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,- token_order, field_prefix,- drop_tokens_threshold, typo_tokens_threshold, exhaustive_search);+ if(query_tokens.size() == 1 && query_tokens[0].value == "*") {+ // synonym can be a wildcard if there was an override filter used+ // we will only do filtering in that case++ if(!syn_wildcard_filter_init_done) {+ curate_filtered_ids(filters, curated_ids, exclude_token_ids, exclude_token_ids_size,+ filter_ids, filter_ids_length, curated_ids_sorted);+ syn_wildcard_filter_init_done = true;+ }++ search_wildcard({"*"}, filters, included_ids_map, sort_fields_std, actual_topster, curated_topster,+ groups_processed, searched_queries, group_limit, group_by_fields,+ curated_ids, curated_ids_sorted,+ exclude_token_ids, exclude_token_ids_size, field_id, field_name,+ all_result_ids, all_result_ids_len, filter_ids, filter_ids_length);+ } else {+ search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,+ field_name, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,+ field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,+ field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,+ token_order, field_prefix,+ drop_tokens_threshold, typo_tokens_threshold, exhaustive_search);+ }}// concat is done only for multi-field searches as `ftopster` will be empty for single-field search@@ -1988,6 +2101,90 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,//LOG(INFO) << "Time taken for result calc: " << timeMillis << "ms";}+void Index::curate_filtered_ids(const std::vector<filter>& filters, const std::set<uint32_t>& curated_ids,+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,+ uint32_t*& filter_ids, uint32_t& filter_ids_length,+ const std::vector<uint32_t>& curated_ids_sorted) const {+ // if filtered results are not available, use the seq_ids index to generate the list of all document ids++ if(filters.empty() && filter_ids_length == 0) {+ filter_ids_length = seq_ids.getLength();+ filter_ids = seq_ids.uncompress();+ }++ if(!curated_ids.empty()) {+ uint32_t *excluded_result_ids = nullptr;+ filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, &curated_ids_sorted[0],+ curated_ids_sorted.size(), &excluded_result_ids);+ delete [] filter_ids;+ filter_ids = excluded_result_ids;+ }++ // Exclude document IDs associated with excluded tokens from the result set+ if(exclude_token_ids_size != 0) {+ uint32_t *excluded_result_ids = nullptr;+ filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, exclude_token_ids,+ exclude_token_ids_size, &excluded_result_ids);+ delete[] filter_ids;+ filter_ids = excluded_result_ids;+ }+}++void Index::search_wildcard(const std::vector<std::string>& qtokens, const std::vector<filter>& filters,+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ const std::vector<sort_by>& sort_fields_std, Topster* topster, Topster* curated_topster,+ spp::sparse_hash_set<uint64_t>& groups_processed,+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,+ const std::vector<std::string>& group_by_fields,+ const std::set<uint32_t>& curated_ids, const std::vector<uint32_t>& curated_ids_sorted,+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size, const uint8_t field_id,+ const string& field, uint32_t*& all_result_ids, size_t& all_result_ids_len,+ const uint32_t* filter_ids, uint32_t filter_ids_length) const {++ // FIXME: duplicated+ int sort_order[3]; // 1 or -1 based on DESC or ASC respectively+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values;+ std::vector<size_t> geopoint_indices;++ for (size_t i = 0; i < sort_fields_std.size(); i++) {+ sort_order[i] = 1;+ if (sort_fields_std[i].order == sort_field_const::asc) {+ sort_order[i] = -1;+ }++ if (sort_fields_std[i].name == sort_field_const::text_match) {+ field_values[i] = &text_match_sentinel_value;+ } else if (sort_fields_std[i].name == sort_field_const::seq_id) {+ field_values[i] = &seq_id_sentinel_value;+ } else if (sort_index.count(sort_fields_std[i].name) != 0) {+ field_values[i] = sort_index.at(sort_fields_std[i].name);++ if (sort_schema.at(sort_fields_std[i].name).is_geopoint()) {+ geopoint_indices.push_back(i);+ }+ }+ }++ uint32_t token_bits = 255;+ std::vector<posting_list_t::iterator_t> plists;++ for(size_t i = 0; i < filter_ids_length; i++) {+ const uint32_t seq_id = filter_ids[i];+ score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, false, 0, topster, {},+ groups_processed, seq_id, sort_order, field_values,+ geopoint_indices, group_limit, group_by_fields, token_bits,+ false, false, plists);+ }++ collate_included_ids(qtokens, field, field_id, included_ids_map, curated_topster, searched_queries);++ uint32_t* new_all_result_ids = nullptr;+ all_result_ids_len = ArrayUtils::or_scalar(all_result_ids, all_result_ids_len, filter_ids,+ filter_ids_length, &new_all_result_ids);+ delete [] all_result_ids;+ all_result_ids = new_all_result_ids;+}+/*1. Split the query into tokens2. Outer loop will generate bounded cartesian product with costs for each tokendiff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex ca33b70d..eff9f8a8 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -262,7 +262,7 @@ TEST_F(CollectionOverrideTest, OverrideJSONValidation) {parse_op = override_t::parse(include_json2, "", override2);ASSERT_FALSE(parse_op.ok());- ASSERT_STREQ("Must contain either `includes` or `excludes`.", parse_op.error().c_str());+ ASSERT_STREQ("Must contain one of:`includes`, `excludes`, `dynamic_filters`.", parse_op.error().c_str());include_json2["includes"] = nlohmann::json::array();include_json2["includes"][0] = 100;@@ -788,4 +788,386 @@ TEST_F(CollectionOverrideTest, PinnedHitsIdsHavingColon) {ASSERT_STREQ("https://example.com/2", res["hits"][9]["document"]["id"].get<std::string>().c_str());collectionManager.drop_collection("coll1");-}\ No newline at end of file+}++TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {+ Collection *coll1;++ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("category", field_types::STRING, true),+ field("brand", field_types::STRING, true),+ field("points", field_types::INT32, false)};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Amazing Shoes";+ doc1["category"] = "shoes";+ doc1["brand"] = "Nike";+ doc1["points"] = 3;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "Track Gym";+ doc2["category"] = "shoes";+ doc2["brand"] = "Adidas";+ doc2["points"] = 5;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());++ std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };++ auto results = coll1->search("shoes", {"name", "category", "brand"}, "",+ {}, sort_fields, {2, 2, 2}, 10).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());++ // with override, results will be different++ nlohmann::json override_json = {+ {"id", "dynamic-filters"},+ {+ "rule", {+ {"query", ".*"},+ {"match", override_t::MATCH_EXACT}+ }+ },+ {"mutate_query_string", true},+ {"dynamic_filters", "category,brand"}+ };++ override_t override;+ auto op = override_t::parse(override_json, "dynamic-filters", override);+ ASSERT_TRUE(op.ok());++ coll1->add_override(override);++ results = coll1->search("shoes", {"name", "category", "brand"}, "",+ {}, sort_fields, {2, 2, 2}, 10).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());++ results = coll1->search("adidas shoes", {"name", "category", "brand"}, "",+ {}, sort_fields, {2, 2, 2}, 10).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());++ // with bad override++ nlohmann::json override_json_bad1 = {+ {"id", "dynamic-filters-bad1"},+ {+ "rule", {+ {"query", ".*"},+ {"match", override_t::MATCH_EXACT}+ }+ },+ {"mutate_query_string", true},+ {"dynamic_filters", ""}+ };++ override_t override_bad1;+ op = override_t::parse(override_json_bad1, "dynamic-filters-bad1", override_bad1);+ ASSERT_FALSE(op.ok());+ ASSERT_EQ("The `dynamic_filters` must be a comma separated string of filter field names.", op.error());++ nlohmann::json override_json_bad2 = {+ {"id", "dynamic-filters-bad2"},+ {+ "rule", {+ {"query", ".*"},+ {"match", override_t::MATCH_EXACT}+ }+ },+ {"mutate_query_string", true},+ {"dynamic_filters", {"foo", "bar"}}+ };++ override_t override_bad2;+ op = override_t::parse(override_json_bad2, "dynamic-filters-bad2", override_bad2);+ ASSERT_FALSE(op.ok());+ ASSERT_EQ("The `dynamic_filters` must be a comma separated string of filter field names.", op.error());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionOverrideTest, DynamicFilteringWithSynonyms) {+ Collection *coll1;++ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("category", field_types::STRING, true),+ field("brand", field_types::STRING, true),+ field("points", field_types::INT32, false)};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Amazing Shoes";+ doc1["category"] = "shoes";+ doc1["brand"] = "Nike";+ doc1["points"] = 3;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "Track Gym";+ doc2["category"] = "shoes";+ doc2["brand"] = "Adidas";+ doc2["points"] = 5;++ nlohmann::json doc3;+ doc3["id"] = "2";+ doc3["name"] = "Amazing Sneakers";+ doc3["category"] = "sneakers";+ doc3["brand"] = "Adidas";+ doc3["points"] = 4;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());++ synonym_t synonym1{"sneakers-shoes", {"sneakers"}, {{"shoes"}} };+ synonym_t synonym2{"boots-shoes", {"boots"}, {{"shoes"}} };+ coll1->add_synonym(synonym1);+ coll1->add_synonym(synonym2);++ std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };++ // with override, results will be different++ nlohmann::json override_json = {+ {"id", "dynamic-filters"},+ {+ "rule", {+ {"query", ".*"},+ {"match", override_t::MATCH_EXACT}+ }+ },+ {"mutate_query_string", true},+ {"dynamic_filters", "category,brand"}+ };++ override_t override;+ auto op = override_t::parse(override_json, "dynamic-filters", override);+ ASSERT_TRUE(op.ok());++ coll1->add_override(override);++ auto results = coll1->search("sneakers", {"name", "category", "brand"}, "",+ {}, sort_fields, {2, 2, 2}, 10).get();++ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][2]["document"]["id"].get<std::string>());++ // keyword does not exist but has a synonym with results++ results = coll1->search("boots", {"name", "category", "brand"}, "",+ {}, sort_fields, {2, 2, 2}, 10).get();++ ASSERT_EQ(2, results["hits"].size());++ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionOverrideTest, StaticFiltering) {+ Collection *coll1;++ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("price", field_types::FLOAT, true),+ field("points", field_types::INT32, false)};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Amazing Shoes";+ doc1["price"] = 399.99;+ doc1["points"] = 3;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "Track Shoes";+ doc2["price"] = 49.99;+ doc2["points"] = 5;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());++ std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };++ nlohmann::json override_json_contains = {+ {"id", "static-filters"},+ {+ "rule", {+ {"query", "expensive"},+ {"match", override_t::MATCH_CONTAINS}+ }+ },+ {"mutate_query_string", true},+ {"dynamic_filters", "price:> 100"}+ };++ override_t override_contains;+ auto op = override_t::parse(override_json_contains, "static-filters", override_contains);+ ASSERT_TRUE(op.ok());++ coll1->add_override(override_contains);++ nlohmann::json override_json_exact = {+ {"id", "static-exact-filters"},+ {+ "rule", {+ {"query", "cheap"},+ {"match", override_t::MATCH_EXACT}+ }+ },+ {"mutate_query_string", true},+ {"dynamic_filters", "price:< 100"}+ };++ override_t override_exact;+ op = override_t::parse(override_json_exact, "static-exact-filters", override_exact);+ ASSERT_TRUE(op.ok());++ coll1->add_override(override_exact);++ auto results = coll1->search("expensive shoes", {"name"}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());++ results = coll1->search("expensive", {"name"}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());++ // with exact match++ results = coll1->search("cheap", {"name"}, "",+ {}, sort_fields, {2}, 10).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());++ // should not work in match contains context++ results = coll1->search("cheap boots", {"name"}, "",+ {}, sort_fields, {2}, 10).get();++ ASSERT_EQ(0, results["hits"].size());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionOverrideTest, FilteringWithAndWithoutQueryStringMutation) {+ Collection *coll1;++ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("price", field_types::FLOAT, true),+ field("points", field_types::INT32, false)};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Apple iPad";+ doc1["price"] = 399.99;+ doc1["points"] = 3;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "Samsung Charger";+ doc2["price"] = 49.99;+ doc2["points"] = 5;++ nlohmann::json doc3;+ doc3["id"] = "2";+ doc3["name"] = "Samsung Phone";+ doc3["price"] = 249.99;+ doc3["points"] = 5;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());++ std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };++ nlohmann::json override_json_contains = {+ {"id", "static-filters"},+ {+ "rule", {+ {"query", "apple"},+ {"match", override_t::MATCH_CONTAINS}+ }+ },+ {"mutate_query_string", false},+ {"dynamic_filters", "price:> 200"}+ };++ override_t override_contains;+ auto op = override_t::parse(override_json_contains, "static-filters", override_contains);+ ASSERT_TRUE(op.ok());++ coll1->add_override(override_contains);++ // first without query string mutation++ auto results = coll1->search("apple", {"name"}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());++ // now, with query string mutation++ override_json_contains = {+ {"id", "static-filters"},+ {+ "rule", {+ {"query", "apple"},+ {"match", override_t::MATCH_CONTAINS}+ }+ },+ {"mutate_query_string", true},+ {"dynamic_filters", "price:> 200"}+ };++ op = override_t::parse(override_json_contains, "static-filters", override_contains);+ ASSERT_TRUE(op.ok());+ coll1->add_override(override_contains);++ results = coll1->search("apple", {"name"}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());++ collectionManager.drop_collection("coll1");+}