Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Dynamic filtering using field interpolation.
Kishore Nallan70059dec15c6
diff --git a/include/collection.h b/include/collection.hindex b1697f2b..2b3baba4 100644--- a/include/collection.h+++ b/include/collection.h@@ -207,7 +207,7 @@ private:const std::map<size_t, std::vector<std::string>>& pinned_hits,const std::vector<std::string>& hidden_hits,std::map<size_t, std::vector<uint32_t>>& include_ids,- std::vector<uint32_t>& excluded_ids, std::vector<const override_t*>& dynamic_overrides,+ std::vector<uint32_t>& excluded_ids, std::vector<const override_t*>& filter_overrides,std::vector<filter>& filters) const;Option<bool> check_and_update_schema(nlohmann::json& document, const DIRTY_VALUES& dirty_values);@@ -228,13 +228,6 @@ private:return std::tie(a_count, a_value_size) > std::tie(b_count, b_value_size);}- Option<bool> parse_filter_query(const std::string& simple_filter_query, std::vector<filter>& filters) const;-- static Option<bool> parse_geopoint_filter_value(std::string& raw_value,- const std::string& format_err_msg,- std::string& processed_filter_val,- NUM_COMPARATOR& num_comparator);-static Option<bool> parse_pinned_hits(const std::string& pinned_hits_str,std::map<size_t, std::vector<std::string>>& pinned_hits);diff --git a/include/field.h b/include/field.hindex a36807d9..756b6692 100644--- a/include/field.h+++ b/include/field.h@@ -6,6 +6,7 @@#include "option.h"#include "string_utils.h"#include "logger.h"+#include <sparsepp.h>#include "json.hpp"namespace field_types {@@ -463,6 +464,17 @@ struct filter {return Option<NUM_COMPARATOR>(num_comparator);}++ static Option<bool> parse_geopoint_filter_value(std::string& raw_value,+ const std::string& format_err_msg,+ std::string& processed_filter_val,+ NUM_COMPARATOR& num_comparator);++ static Option<bool> parse_filter_query(const std::string& simple_filter_query,+ const std::unordered_map<std::string, field>& search_schema,+ Store* store,+ const std::string& doc_id_prefix,+ std::vector<filter>& filters);};namespace sort_field_const {diff --git a/include/index.h b/include/index.hindex 8f2453b1..e8edf62a 100644--- a/include/index.h+++ b/include/index.h@@ -54,6 +54,7 @@ struct override_t {struct rule_t {std::string query;std::string match;+ bool dynamic_query = false;};struct add_hit_t {@@ -71,7 +72,7 @@ struct override_t {std::vector<add_hit_t> add_hits;std::vector<drop_hit_t> drop_hits;- std::vector<std::string> dynamic_filters;+ std::string filter_by;bool mutate_query_string = false;override_t() {}@@ -90,8 +91,8 @@ struct override_t {}if(override_json.count("includes") == 0 && override_json.count("excludes") == 0 &&- override_json.count("dynamic_filters") == 0) {- return Option<bool>(400, "Must contain one of:`includes`, `excludes`, `dynamic_filters`.");+ override_json.count("filter_by") == 0) {+ return Option<bool>(400, "Must contain one of:`includes`, `excludes`, `filter_by`.");}if(override_json.count("includes") != 0) {@@ -139,15 +140,13 @@ struct override_t {}- if(override_json.count("dynamic_filters") != 0) {- if(!override_json["dynamic_filters"].is_string()) {- return Option<bool>(400, "The `dynamic_filters` must be "- "a comma separated string of filter field names.");+ if(override_json.count("filter_by") != 0) {+ if(!override_json["filter_by"].is_string()) {+ return Option<bool>(400, "The `filter_by` must be a string.");}- if(override_json["dynamic_filters"].get<std::string>().empty()) {- return Option<bool>(400, "The `dynamic_filters` must be "- "a comma separated string of filter field names.");+ if(override_json["filter_by"].get<std::string>().empty()) {+ return Option<bool>(400, "The `filter_by` must be a non-empty string.");}}@@ -187,10 +186,25 @@ struct override_t {}}- if (override_json.count("dynamic_filters") != 0) {- std::vector<std::string> filter_fields;- StringUtils::split(override_json["dynamic_filters"].get<std::string>(), filter_fields, ",");- override.dynamic_filters = filter_fields;+ if (override_json.count("filter_by") != 0) {+ override.filter_by = override_json["filter_by"].get<std::string>();+ }++ // we have to also detect if it is a dynamic query rule+ size_t i = 0;+ while(i < override.rule.query.size()) {+ if(override.rule.query[i] == '{') {+ // look for closing curly+ i++;+ while(i < override.rule.query.size()) {+ if(override.rule.query[i] == '}') {+ override.rule.dynamic_query = true;+ break;+ }+ i++;+ }+ }+ i++;}return Option<bool>(true);@@ -246,7 +260,7 @@ struct search_args {size_t all_result_ids_len;bool exhaustive_search;size_t concurrency;- const std::vector<const override_t*>& dynamic_overrides;+ const std::vector<const override_t*>& filter_overrides;spp::sparse_hash_set<uint64_t> groups_processed;std::vector<std::vector<art_leaf*>> searched_queries;@@ -278,7 +292,7 @@ struct search_args {group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),prioritize_exact_match(prioritize_exact_match), all_result_ids_len(0),exhaustive_search(exhaustive_search), concurrency(concurrency),- dynamic_overrides(dynamic_overrides) {+ filter_overrides(dynamic_overrides) {const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatorytopster = new Topster(topster_size, group_limit);@@ -347,6 +361,10 @@ private:std::string name;+ const uint32_t collection_id;++ Store* store;+size_t num_documents;std::unordered_map<std::string, field> search_schema;@@ -397,12 +415,26 @@ private:size_t group_limit, const std::vector<std::string>& group_by_fields,const uint32_t* result_ids, size_t results_size) const;- void process_dynamic_overrides(const std::vector<const override_t*>& dynamic_overrides,+ void static_filter_query_eval(const override_t* override, std::vector<std::string>& tokens,+ std::vector<filter>& filters) const;++ void process_filter_overrides(const std::vector<const override_t*>& filter_overrides,std::vector<query_tokens_t>& field_query_tokens,- const token_ordering token_order,+ token_ordering token_order,+ std::vector<filter>& filters,uint32_t** filter_ids,uint32_t& filter_ids_length) const;+ bool resolve_override(const std::vector<std::string>& rule_parts, const bool exact_rule_match,+ const std::vector<std::string>& query_tokens,+ token_ordering token_order, std::set<std::string>& absorbed_tokens,+ uint32_t*& override_ids, size_t& override_ids_len) const;++ bool check_for_overrides2(const token_ordering& token_order, const string& field_name, const bool slide_window,+ uint32_t*& field_override_ids, size_t& field_override_ids_len,+ bool exact_rule_match, std::vector<std::string>& tokens,+ std::set<std::string>& absorbed_tokens) const;+void search_field(const uint8_t & field_id,std::vector<token_t>& query_tokens,std::vector<token_t>& search_tokens,@@ -510,6 +542,8 @@ public:Index() = delete;Index(const std::string& name,+ const uint32_t collection_id,+ const Store* store,ThreadPool* thread_pool,const std::unordered_map<std::string, field>& search_schema,const std::map<std::string, field>& facet_schema,@@ -563,7 +597,7 @@ public:void search(std::vector<query_tokens_t>& field_query_tokens,const std::vector<search_field_t> & search_fields,- const std::vector<filter> & filters, std::vector<facet> & facets,+ std::vector<filter> & filters, std::vector<facet> & facets,facet_query_t & facet_query,const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,const std::vector<uint32_t> & excluded_ids,@@ -579,7 +613,7 @@ public:const size_t typo_tokens_threshold,const size_t group_limit,const std::vector<std::string>& group_by_fields,- const std::vector<const override_t*>& dynamic_overrides,+ const std::vector<const override_t*>& filter_overrides,const std::string& default_sorting_field,bool prioritize_exact_match,bool exhaustive_search,@@ -606,7 +640,7 @@ public:art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);- uint32_t do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) const;+ void do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length, const std::vector<filter>& filters) const;void refresh_schemas(const std::vector<field>& new_fields);@@ -622,10 +656,6 @@ public:const std::string& fallback_field_type,const DIRTY_VALUES& dirty_values);- void check_for_overrides(const token_ordering& token_order, const string& field_name, bool mutate_query_string,- uint32_t*& field_override_ids,- size_t& field_override_ids_len, std::vector<std::string>& tokens) const;-void search_wildcard(const std::vector<std::string>& qtokens, const std::vector<filter>& filters,const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,const std::vector<sort_by>& sort_fields_std, Topster* topster, Topster* curated_topster,diff --git a/src/collection.cpp b/src/collection.cppindex f277c5d3..19bc6638 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -377,7 +377,7 @@ void Collection::curate_results(string& actual_query, bool enable_overrides, booconst std::map<size_t, std::vector<std::string>>& pinned_hits,const std::vector<std::string>& hidden_hits,std::map<size_t, std::vector<uint32_t>>& include_ids,- std::vector<uint32_t>& excluded_ids, std::vector<const override_t*>& dynamic_overrides,+ std::vector<uint32_t>& excluded_ids, std::vector<const override_t*>& filter_overrides,std::vector<filter>& filters) const {std::set<uint32_t> excluded_set;@@ -403,9 +403,9 @@ void Collection::curate_results(string& actual_query, bool enable_overrides, boofor(const auto& override_kv: overrides) {const auto& override = override_kv.second;- // static overrides are applied first as they precedence over dynamic overrides- if(!override.dynamic_filters.empty() && override.rule.query == ".*") {- dynamic_overrides.push_back(&override);+ // ID-based overrides are applied first as they take precedence over filter-based overrides+ if(!override.filter_by.empty()) {+ filter_overrides.push_back(&override);continue;}@@ -432,21 +432,6 @@ void Collection::curate_results(string& actual_query, bool enable_overrides, booinclude_ids[hit.position].push_back(seq_id);}}-- for(const auto& filter_str: override.dynamic_filters) {- auto op = parse_filter_query(filter_str, filters);- if(op.ok() && override.mutate_query_string) {- actual_query.replace(- actual_query.find(override.rule.query),- override.rule.query.size(),- ""- );-- if(StringUtils::trim(actual_query).empty()) {- actual_query = "*";- }- }- }}}}@@ -587,8 +572,10 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s}}+ const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";std::vector<filter> filters;- Option<bool> parse_filter_op = parse_filter_query(simple_filter_query, filters);+ Option<bool> parse_filter_op = filter::parse_filter_query(simple_filter_query, search_schema,+ store, doc_id_prefix, filters);if(!parse_filter_op.ok()) {return Option<nlohmann::json>(parse_filter_op.code(), parse_filter_op.error());}@@ -840,10 +827,10 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sstd::vector<std::string> hidden_hits;StringUtils::split(hidden_hits_str, hidden_hits, ",");- std::vector<const override_t*> dynamic_overrides;+ std::vector<const override_t*> filter_overrides;std::string query = raw_query;curate_results(query, enable_overrides, pre_segmented_query, pinned_hits, hidden_hits,- include_ids, excluded_ids, dynamic_overrides, filters);+ include_ids, excluded_ids, filter_overrides, filters);/*for(auto& kv: include_ids) {LOG(INFO) << "key: " << kv.first;@@ -911,12 +898,12 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const ssize_t index_id = 0;search_args* search_params = new search_args(field_query_tokens, weighted_search_fields,- filters, facets, included_ids, excluded_ids,- sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,- per_page, page, token_order, prefixes,- drop_tokens_threshold, typo_tokens_threshold,- group_by_fields, group_limit, default_sorting_field, prioritize_exact_match,- exhaustive_search, 4, dynamic_overrides);+ filters, facets, included_ids, excluded_ids,+ sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,+ per_page, page, token_order, prefixes,+ drop_tokens_threshold, typo_tokens_threshold,+ group_by_fields, group_limit, default_sorting_field, prioritize_exact_match,+ exhaustive_search, 4, filter_overrides);index->run_search(search_params);@@ -1361,15 +1348,18 @@ Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,std::vector<std::pair<size_t, uint32_t*>>& index_ids) {std::shared_lock lock(mutex);+ const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";std::vector<filter> filters;- Option<bool> filter_op = parse_filter_query(simple_filter_query, filters);+ Option<bool> filter_op = filter::parse_filter_query(simple_filter_query, search_schema,+ store, doc_id_prefix, filters);if(!filter_op.ok()) {return filter_op;}uint32_t* filter_ids = nullptr;- size_t filter_ids_len = index->do_filtering(&filter_ids, filters);+ uint32_t filter_ids_len = 0;+ index->do_filtering(filter_ids, filter_ids_len, filters);index_ids.emplace_back(filter_ids_len, filter_ids);return Option<bool>(true);@@ -2025,346 +2015,6 @@ const Index* Collection::_get_index() const {return index;}-Option<bool> Collection::parse_geopoint_filter_value(std::string& raw_value,- const std::string& format_err_msg,- std::string& processed_filter_val,- NUM_COMPARATOR& num_comparator) {-- num_comparator = LESS_THAN_EQUALS;-- if(!(raw_value[0] == '(' && raw_value[raw_value.size() - 1] == ')')) {- return Option<bool>(400, format_err_msg);- }-- std::vector<std::string> filter_values;- auto raw_val_without_paran = raw_value.substr(1, raw_value.size() - 2);- StringUtils::split(raw_val_without_paran, filter_values, ",");-- // we will end up with: "10.45 34.56 2 km" or a geo polygon-- if(filter_values.size() < 3) {- return Option<bool>(400, format_err_msg);- }-- // do validation: format should match either a point + radius or polygon-- size_t num_floats = 0;- for(const auto& fvalue: filter_values) {- if(StringUtils::is_float(fvalue)) {- num_floats++;- }- }-- bool is_polygon = (num_floats == filter_values.size());- if(!is_polygon) {- // we have to ensure that this is a point + radius match- if(!StringUtils::is_float(filter_values[0]) || !StringUtils::is_float(filter_values[1])) {- return Option<bool>(400, format_err_msg);- }- }-- if(is_polygon) {- processed_filter_val = raw_val_without_paran;- } else {- // point + radius- std::vector<std::string> dist_values;- StringUtils::split(filter_values[2], dist_values, " ");-- if(dist_values.size() != 2) {- return Option<bool>(400, format_err_msg);- }-- if(dist_values[1] != "km" && dist_values[1] != "mi") {- return Option<bool>(400, "Unit must be either `km` or `mi`.");- }-- processed_filter_val = filter_values[0] + ", " + filter_values[1] + ", " + // co-ords- dist_values[0] + ", " + dist_values[1]; // X km- }-- return Option<bool>(true);-}--Option<bool> Collection::parse_filter_query(const std::string& simple_filter_query,- std::vector<filter>& filters) const {-- std::vector<filter> exclude_filters; // to ensure that they go last in the list of filters-- std::vector<std::string> filter_blocks;- StringUtils::split(simple_filter_query, filter_blocks, "&&");-- for(const std::string & filter_block: filter_blocks) {- // split into [field_name, value]- size_t found_index = filter_block.find(':');-- if(found_index == std::string::npos) {- return Option<bool>(400, "Could not parse the filter query.");- }-- std::string&& field_name = filter_block.substr(0, found_index);- StringUtils::trim(field_name);-- if(field_name == "id") {- filter id_filter;- std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);- StringUtils::trim(raw_value);- id_filter = {field_name, {}, {}};-- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {- std::vector<std::string> doc_ids;- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), doc_ids, ",");-- for(std::string& doc_id: doc_ids) {- // we have to convert the doc_id to seq id- std::string seq_id_str;- StoreStatus seq_id_status = store->get(get_doc_id_key(doc_id), seq_id_str);-- if(seq_id_status != StoreStatus::FOUND) {- continue;- }-- id_filter.values.push_back(seq_id_str);- id_filter.comparators.push_back(EQUALS);- }- } else {- std::string seq_id_str;- StoreStatus seq_id_status = store->get(get_doc_id_key(raw_value), seq_id_str);- if(seq_id_status == StoreStatus::FOUND) {- id_filter.values.push_back(seq_id_str);- id_filter.comparators.push_back(EQUALS);- }- }-- if(!id_filter.values.empty()) {- filters.push_back(id_filter);- }-- continue;- }-- if(search_schema.count(field_name) == 0) {- return Option<bool>(404, "Could not find a filter field named `" + field_name + "` in the schema.");- }-- field _field = search_schema.at(field_name);- std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);- StringUtils::trim(raw_value);- filter f;-- // skip past optional `:=` operator, which has no meaning for non-string fields- if(!_field.is_string() && raw_value[0] == '=') {- size_t filter_value_index = 0;- while(raw_value[++filter_value_index] == ' ');- raw_value = raw_value.substr(filter_value_index);- }-- if(_field.is_integer() || _field.is_float()) {- // could be a single value or a list- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {- std::vector<std::string> filter_values;- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");-- f = {field_name, {}, {}};-- for(std::string & filter_value: filter_values) {- Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(filter_value);- if(!op_comparator.ok()) {- return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());- }-- if(op_comparator.get() == RANGE_INCLUSIVE) {- // split the value around range operator to extract bounds- std::vector<std::string> range_values;- StringUtils::split(filter_value, range_values, filter::RANGE_OPERATOR());- for(const std::string& range_value: range_values) {- auto validate_op = filter::validate_numerical_filter_value(_field, range_value);- if(!validate_op.ok()) {- return validate_op;- }-- f.values.push_back(range_value);- f.comparators.push_back(op_comparator.get());- }- } else {- auto validate_op = filter::validate_numerical_filter_value(_field, filter_value);- if(!validate_op.ok()) {- return validate_op;- }-- f.values.push_back(filter_value);- f.comparators.push_back(op_comparator.get());- }- }-- } else {- Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);- if(!op_comparator.ok()) {- return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());- }-- if(op_comparator.get() == RANGE_INCLUSIVE) {- // split the value around range operator to extract bounds- std::vector<std::string> range_values;- StringUtils::split(raw_value, range_values, filter::RANGE_OPERATOR());-- f.field_name = field_name;- for(const std::string& range_value: range_values) {- auto validate_op = filter::validate_numerical_filter_value(_field, range_value);- if(!validate_op.ok()) {- return validate_op;- }-- f.values.push_back(range_value);- f.comparators.push_back(op_comparator.get());- }- } else {- auto validate_op = filter::validate_numerical_filter_value(_field, raw_value);- if(!validate_op.ok()) {- return validate_op;- }- f = {field_name, {raw_value}, {op_comparator.get()}};- }- }- } else if(_field.is_bool()) {- NUM_COMPARATOR bool_comparator = EQUALS;- size_t filter_value_index = 0;-- if(raw_value[0] == '=') {- bool_comparator = EQUALS;- while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');- } else if(raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {- bool_comparator = NOT_EQUALS;- filter_value_index++;- while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');- }-- if(filter_value_index != 0) {- raw_value = raw_value.substr(filter_value_index);- }-- if(filter_value_index == raw_value.size()) {- return Option<bool>(400, "Error with filter field `" + _field.name +- "`: Filter value cannot be empty.");- }-- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {- std::vector<std::string> filter_values;- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");- f = {field_name, {}, {}};-- for(std::string & filter_value: filter_values) {- if(filter_value != "true" && filter_value != "false") {- return Option<bool>(400, "Values of filter field `" + _field.name +- "`: must be `true` or `false`.");- }-- filter_value = (filter_value == "true") ? "1" : "0";- f.values.push_back(filter_value);- f.comparators.push_back(bool_comparator);- }- } else {- if(raw_value != "true" && raw_value != "false") {- return Option<bool>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");- }-- std::string bool_value = (raw_value == "true") ? "1" : "0";- f = {field_name, {bool_value}, {bool_comparator}};- }-- } else if(_field.is_geopoint()) {- f = {field_name, {}, {}};-- const std::string& format_err_msg = "Value of filter field `" + _field.name +- "`: must be in the `(-44.50, 170.29, 0.75 km)` or "- "(56.33, -65.97, 23.82, -127.82) format.";-- NUM_COMPARATOR num_comparator;-- // could be a single value or a list- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {- std::vector<std::string> filter_values;- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, "),");-- for(std::string& filter_value: filter_values) {- filter_value += ")";- std::string processed_filter_val;- auto parse_op = parse_geopoint_filter_value(filter_value, format_err_msg, processed_filter_val, num_comparator);-- if(!parse_op.ok()) {- return parse_op;- }-- f.values.push_back(processed_filter_val);- f.comparators.push_back(num_comparator);- }- } else {- // single value, e.g. (10.45, 34.56, 2 km)- std::string processed_filter_val;- auto parse_op = parse_geopoint_filter_value(raw_value, format_err_msg, processed_filter_val, num_comparator);-- if(!parse_op.ok()) {- return parse_op;- }-- f.values.push_back(processed_filter_val);- f.comparators.push_back(num_comparator);- }-- } else if(_field.is_string()) {- size_t filter_value_index = 0;- NUM_COMPARATOR str_comparator = CONTAINS;-- if(raw_value[0] == '=') {- if(!_field.facet) {- // EQUALS filtering on string is possible only on facet fields- return Option<bool>(400, "To perform exact filtering, filter field `" +- _field.name + "` must be a facet field.");- }-- // string filter should be evaluated in strict "equals" mode- str_comparator = EQUALS;- while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');- } else if(raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {- if(!_field.facet) {- // EXCLUDE filtering on string is possible only on facet fields- return Option<bool>(400, "To perform exclude filtering, filter field `" +- _field.name + "` must be a facet field.");- }-- str_comparator = NOT_EQUALS;- filter_value_index++;- while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');- }-- if(filter_value_index == raw_value.size()) {- return Option<bool>(400, "Error with filter field `" + _field.name +- "`: Filter value cannot be empty.");- }-- if(raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {- std::vector<std::string> filter_values;- StringUtils::split_to_values(raw_value.substr(filter_value_index+1, raw_value.size() - filter_value_index - 2), filter_values);- f = {field_name, filter_values, {str_comparator}};- } else {- f = {field_name, {raw_value.substr(filter_value_index)}, {str_comparator}};- }- } else {- return Option<bool>(400, "Error with filter field `" + _field.name +- "`: Unidentified field data type, see docs for supported data types.");- }-- if(f.comparators.size() > 0 && f.comparators.front() == NOT_EQUALS) {- exclude_filters.push_back(f);- } else {- filters.push_back(f);- }- }-- filters.insert( filters.end(), exclude_filters.begin(), exclude_filters.end() );-- return Option<bool>(true);-}-Option<bool> Collection::parse_pinned_hits(const std::string& pinned_hits_str,std::map<size_t, std::vector<std::string>>& pinned_hits) {if(!pinned_hits_str.empty()) {@@ -2721,6 +2371,8 @@ Index* Collection::init_index() {}return new Index(name+std::to_string(0),+ collection_id,+ store,CollectionManager::get_instance().get_thread_pool(),search_schema, facet_schema, sort_schema,symbols_to_index, token_separators);diff --git a/src/field.cpp b/src/field.cppnew file mode 100644index 00000000..b0ba565e--- /dev/null+++ b/src/field.cpp@@ -0,0 +1,345 @@+#include <store.h>+#include "field.h"++Option<bool> filter::parse_geopoint_filter_value(std::string& raw_value,+ const std::string& format_err_msg,+ std::string& processed_filter_val,+ NUM_COMPARATOR& num_comparator) {++ num_comparator = LESS_THAN_EQUALS;++ if(!(raw_value[0] == '(' && raw_value[raw_value.size() - 1] == ')')) {+ return Option<bool>(400, format_err_msg);+ }++ std::vector<std::string> filter_values;+ auto raw_val_without_paran = raw_value.substr(1, raw_value.size() - 2);+ StringUtils::split(raw_val_without_paran, filter_values, ",");++ // we will end up with: "10.45 34.56 2 km" or a geo polygon++ if(filter_values.size() < 3) {+ return Option<bool>(400, format_err_msg);+ }++ // do validation: format should match either a point + radius or polygon++ size_t num_floats = 0;+ for(const auto& fvalue: filter_values) {+ if(StringUtils::is_float(fvalue)) {+ num_floats++;+ }+ }++ bool is_polygon = (num_floats == filter_values.size());+ if(!is_polygon) {+ // we have to ensure that this is a point + radius match+ if(!StringUtils::is_float(filter_values[0]) || !StringUtils::is_float(filter_values[1])) {+ return Option<bool>(400, format_err_msg);+ }+ }++ if(is_polygon) {+ processed_filter_val = raw_val_without_paran;+ } else {+ // point + radius+ std::vector<std::string> dist_values;+ StringUtils::split(filter_values[2], dist_values, " ");++ if(dist_values.size() != 2) {+ return Option<bool>(400, format_err_msg);+ }++ if(dist_values[1] != "km" && dist_values[1] != "mi") {+ return Option<bool>(400, "Unit must be either `km` or `mi`.");+ }++ processed_filter_val = filter_values[0] + ", " + filter_values[1] + ", " + // co-ords+ dist_values[0] + ", " + dist_values[1]; // X km+ }++ return Option<bool>(true);+}++Option<bool> filter::parse_filter_query(const string& simple_filter_query,+ const std::unordered_map<std::string, field>& search_schema,+ Store* store,+ const std::string& doc_id_prefix,+ std::vector<filter>& filters) {++ std::vector<filter> exclude_filters; // to ensure that they go last in the list of filters++ std::vector<std::string> filter_blocks;+ StringUtils::split(simple_filter_query, filter_blocks, "&&");++ for(const std::string & filter_block: filter_blocks) {+ // split into [field_name, value]+ size_t found_index = filter_block.find(':');++ if(found_index == std::string::npos) {+ return Option<bool>(400, "Could not parse the filter query.");+ }++ std::string&& field_name = filter_block.substr(0, found_index);+ StringUtils::trim(field_name);++ if(field_name == "id") {+ filter id_filter;+ std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);+ StringUtils::trim(raw_value);+ id_filter = {field_name, {}, {}};++ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {+ std::vector<std::string> doc_ids;+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), doc_ids, ",");++ for(std::string& doc_id: doc_ids) {+ // we have to convert the doc_id to seq id+ std::string seq_id_str;+ StoreStatus seq_id_status = store->get(doc_id_prefix + doc_id, seq_id_str);++ if(seq_id_status != StoreStatus::FOUND) {+ continue;+ }++ id_filter.values.push_back(seq_id_str);+ id_filter.comparators.push_back(EQUALS);+ }+ } else {+ std::string seq_id_str;+ StoreStatus seq_id_status = store->get(doc_id_prefix + raw_value, seq_id_str);+ if(seq_id_status == StoreStatus::FOUND) {+ id_filter.values.push_back(seq_id_str);+ id_filter.comparators.push_back(EQUALS);+ }+ }++ if(!id_filter.values.empty()) {+ filters.push_back(id_filter);+ }++ continue;+ }++ if(search_schema.count(field_name) == 0) {+ return Option<bool>(404, "Could not find a filter field named `" + field_name + "` in the schema.");+ }++ field _field = search_schema.at(field_name);+ std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);+ StringUtils::trim(raw_value);+ filter f;++ // skip past optional `:=` operator, which has no meaning for non-string fields+ if(!_field.is_string() && raw_value[0] == '=') {+ size_t filter_value_index = 0;+ while(raw_value[++filter_value_index] == ' ');+ raw_value = raw_value.substr(filter_value_index);+ }++ if(_field.is_integer() || _field.is_float()) {+ // could be a single value or a list+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {+ std::vector<std::string> filter_values;+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");++ f = {field_name, {}, {}};++ for(std::string & filter_value: filter_values) {+ Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(filter_value);+ if(!op_comparator.ok()) {+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());+ }++ if(op_comparator.get() == RANGE_INCLUSIVE) {+ // split the value around range operator to extract bounds+ std::vector<std::string> range_values;+ StringUtils::split(filter_value, range_values, filter::RANGE_OPERATOR());+ for(const std::string& range_value: range_values) {+ auto validate_op = filter::validate_numerical_filter_value(_field, range_value);+ if(!validate_op.ok()) {+ return validate_op;+ }++ f.values.push_back(range_value);+ f.comparators.push_back(op_comparator.get());+ }+ } else {+ auto validate_op = filter::validate_numerical_filter_value(_field, filter_value);+ if(!validate_op.ok()) {+ return validate_op;+ }++ f.values.push_back(filter_value);+ f.comparators.push_back(op_comparator.get());+ }+ }++ } else {+ Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);+ if(!op_comparator.ok()) {+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());+ }++ if(op_comparator.get() == RANGE_INCLUSIVE) {+ // split the value around range operator to extract bounds+ std::vector<std::string> range_values;+ StringUtils::split(raw_value, range_values, filter::RANGE_OPERATOR());++ f.field_name = field_name;+ for(const std::string& range_value: range_values) {+ auto validate_op = filter::validate_numerical_filter_value(_field, range_value);+ if(!validate_op.ok()) {+ return validate_op;+ }++ f.values.push_back(range_value);+ f.comparators.push_back(op_comparator.get());+ }+ } else {+ auto validate_op = filter::validate_numerical_filter_value(_field, raw_value);+ if(!validate_op.ok()) {+ return validate_op;+ }+ f = {field_name, {raw_value}, {op_comparator.get()}};+ }+ }+ } else if(_field.is_bool()) {+ NUM_COMPARATOR bool_comparator = EQUALS;+ size_t filter_value_index = 0;++ if(raw_value[0] == '=') {+ bool_comparator = EQUALS;+ while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');+ } else if(raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {+ bool_comparator = NOT_EQUALS;+ filter_value_index++;+ while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');+ }++ if(filter_value_index != 0) {+ raw_value = raw_value.substr(filter_value_index);+ }++ if(filter_value_index == raw_value.size()) {+ return Option<bool>(400, "Error with filter field `" + _field.name ++ "`: Filter value cannot be empty.");+ }++ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {+ std::vector<std::string> filter_values;+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");+ f = {field_name, {}, {}};++ for(std::string & filter_value: filter_values) {+ if(filter_value != "true" && filter_value != "false") {+ return Option<bool>(400, "Values of filter field `" + _field.name ++ "`: must be `true` or `false`.");+ }++ filter_value = (filter_value == "true") ? "1" : "0";+ f.values.push_back(filter_value);+ f.comparators.push_back(bool_comparator);+ }+ } else {+ if(raw_value != "true" && raw_value != "false") {+ return Option<bool>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");+ }++ std::string bool_value = (raw_value == "true") ? "1" : "0";+ f = {field_name, {bool_value}, {bool_comparator}};+ }++ } else if(_field.is_geopoint()) {+ f = {field_name, {}, {}};++ const std::string& format_err_msg = "Value of filter field `" + _field.name ++ "`: must be in the `(-44.50, 170.29, 0.75 km)` or "+ "(56.33, -65.97, 23.82, -127.82) format.";++ NUM_COMPARATOR num_comparator;++ // could be a single value or a list+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {+ std::vector<std::string> filter_values;+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, "),");++ for(std::string& filter_value: filter_values) {+ filter_value += ")";+ std::string processed_filter_val;+ auto parse_op = parse_geopoint_filter_value(filter_value, format_err_msg, processed_filter_val, num_comparator);++ if(!parse_op.ok()) {+ return parse_op;+ }++ f.values.push_back(processed_filter_val);+ f.comparators.push_back(num_comparator);+ }+ } else {+ // single value, e.g. (10.45, 34.56, 2 km)+ std::string processed_filter_val;+ auto parse_op = parse_geopoint_filter_value(raw_value, format_err_msg, processed_filter_val, num_comparator);++ if(!parse_op.ok()) {+ return parse_op;+ }++ f.values.push_back(processed_filter_val);+ f.comparators.push_back(num_comparator);+ }++ } else if(_field.is_string()) {+ size_t filter_value_index = 0;+ NUM_COMPARATOR str_comparator = CONTAINS;++ if(raw_value[0] == '=') {+ if(!_field.facet) {+ // EQUALS filtering on string is possible only on facet fields+ return Option<bool>(400, "To perform exact filtering, filter field `" ++ _field.name + "` must be a facet field.");+ }++ // string filter should be evaluated in strict "equals" mode+ str_comparator = EQUALS;+ while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');+ } else if(raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {+ if(!_field.facet) {+ // EXCLUDE filtering on string is possible only on facet fields+ return Option<bool>(400, "To perform exclude filtering, filter field `" ++ _field.name + "` must be a facet field.");+ }++ str_comparator = NOT_EQUALS;+ filter_value_index++;+ while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');+ }++ if(filter_value_index == raw_value.size()) {+ return Option<bool>(400, "Error with filter field `" + _field.name ++ "`: Filter value cannot be empty.");+ }++ if(raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {+ std::vector<std::string> filter_values;+ StringUtils::split_to_values(raw_value.substr(filter_value_index+1, raw_value.size() - filter_value_index - 2), filter_values);+ f = {field_name, filter_values, {str_comparator}};+ } else {+ f = {field_name, {raw_value.substr(filter_value_index)}, {str_comparator}};+ }+ } else {+ return Option<bool>(400, "Error with filter field `" + _field.name ++ "`: Unidentified field data type, see docs for supported data types.");+ }++ if(f.comparators.size() > 0 && f.comparators.front() == NOT_EQUALS) {+ exclude_filters.push_back(f);+ } else {+ filters.push_back(f);+ }+ }++ filters.insert( filters.end(), exclude_filters.begin(), exclude_filters.end() );++ return Option<bool>(true);+}diff --git a/src/index.cpp b/src/index.cppindex c61f9981..9c81aca4 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -24,11 +24,11 @@ spp::sparse_hash_map<uint32_t, int64_t> Index::text_match_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t> Index::seq_id_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t> Index::geo_sentinel_value;-Index::Index(const std::string& name, ThreadPool* thread_pool,+Index::Index(const std::string& name, const uint32_t collection_id, const Store* store, ThreadPool* thread_pool,const std::unordered_map<std::string, field> & search_schema,const std::map<std::string, field>& facet_schema, const std::unordered_map<std::string, field>& sort_schema,const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators):- name(name), thread_pool(thread_pool),+ name(name), collection_id(collection_id), thread_pool(thread_pool),search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema),symbols_to_index(symbols_to_index), token_separators(token_separators) {@@ -1071,13 +1071,11 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,}}-uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) const {+void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,+ const std::vector<filter>& filters) const {//auto begin = std::chrono::high_resolution_clock::now();std::shared_lock lock(mutex);- uint32_t* filter_ids = nullptr;- uint32_t filter_ids_length = 0;-for(size_t i = 0; i < filters.size(); i++) {const filter & a_filter = filters[i];@@ -1422,9 +1420,6 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filterstd::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();LOG(INFO) << "Time taken for filtering: " << timeMillis << "ms";*/-- *filter_ids_out = filter_ids;- return filter_ids_length;}void Index::run_search(search_args* search_params) {@@ -1441,7 +1436,7 @@ void Index::run_search(search_args* search_params) {search_params->raw_result_kvs, search_params->override_result_kvs,search_params->typo_tokens_threshold,search_params->group_limit, search_params->group_by_fields,- search_params->dynamic_overrides,+ search_params->filter_overrides,search_params->default_sorting_field,search_params->prioritize_exact_match,search_params->exhaustive_search,@@ -1521,49 +1516,193 @@ void Index::concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t,}}-void Index::process_dynamic_overrides(const std::vector<const override_t*>& dynamic_overrides,- std::vector<query_tokens_t>& field_query_tokens,- token_ordering token_order,- uint32_t** filter_ids,- uint32_t& filter_ids_length) const {-- // compute dynamic filters if any- for(auto& override: dynamic_overrides) {- // will do an AND across different fields- uint32_t* field_override_ids = nullptr;- size_t field_override_ids_len = 0;-- // check for dynamic filter- for(auto& field_name: override->dynamic_filters) {- // check if query matches a filter field- std::vector<std::string> tokens = field_query_tokens[0].q_include_tokens;- check_for_overrides(token_order, field_name, override->mutate_query_string,- field_override_ids, field_override_ids_len, tokens);-- for(size_t i=0; i < field_query_tokens.size(); i++) {- if(tokens.empty()) {- field_query_tokens[i].q_include_tokens = {"*"};- } else {- field_query_tokens[i].q_include_tokens = tokens;+void Index::static_filter_query_eval(const override_t* override,+ std::vector<std::string>& tokens,+ std::vector<filter>& filters) const {++ std::string query = StringUtils::join(tokens, " ");++ if( (override->rule.match == override_t::MATCH_EXACT && override->rule.query == query) ||+ (override->rule.match == override_t::MATCH_CONTAINS && query.find(override->rule.query) != std::string::npos) ) {++ Option<bool> filter_op = filter::parse_filter_query(override->filter_by, search_schema,+ store, "", filters);+ if(!filter_op.ok()) {+ return ;+ }++ if(override->mutate_query_string) {+ query.replace(query.find(override->rule.query), override->rule.query.size(), "");+ if(StringUtils::trim(query).empty()) {+ tokens = {"*"};+ } else {+ tokens.clear();+ StringUtils::split(query, tokens, " ");+ }+ }+ }+}++bool Index::resolve_override(const std::vector<std::string>& rule_parts, const bool exact_rule_match,+ const std::vector<std::string>& query_tokens,+ token_ordering token_order, std::set<std::string>& absorbed_tokens,+ uint32_t*& override_ids, size_t& override_ids_len) const {++ bool resolved_override = false;+ size_t i = 0, j = 0;++ while(i < rule_parts.size()) {+ if(rule_parts[i].front() == '{' && rule_parts[i].back() == '}') {+ // found a field placeholder+ std::vector<std::string> field_names;+ std::string rule_part = rule_parts[i];+ field_names.emplace_back(rule_part.erase(0, 1).erase(rule_part.size() - 1));++ // skip until we find a non-placeholder token+ i++;++ while(i < rule_parts.size() && (rule_parts[i].front() == '{' && rule_parts[i].back() == '}')) {+ rule_part = rule_parts[i];+ field_names.emplace_back(rule_part.erase(0, 1).erase(rule_part.size() - 1));+ i++;+ }++ // `i` now points to either end of array or at a non-placeholder rule token+ // we will skip query tokens until it matches the rule token++ std::vector<std::string> matched_tokens;++ while(j < query_tokens.size() && (i == rule_parts.size() || rule_parts[i] != query_tokens[j])) {+ matched_tokens.emplace_back(query_tokens[j]);+ j++;+ }++ if(i < rule_parts.size() && j < query_tokens.size() && rule_parts[i] != query_tokens[j]) {+ // if last token does not match, it means that the query does not match the rule+ resolved_override = false;+ goto RETURN_EARLY;+ }++ if(field_names.empty()) {+ resolved_override = false;+ goto RETURN_EARLY;+ }++ resolved_override = true;++ // we try to map `field_names` against `matched_tokens` now+ for(size_t findex = 0; findex < field_names.size(); findex++) {+ const auto& field_name = field_names[findex];+ bool slide_window = (findex == 0); // fields following another field should match exactly+ resolved_override &= check_for_overrides2(token_order, field_name, slide_window,+ override_ids, override_ids_len,+ exact_rule_match, matched_tokens, absorbed_tokens);+ if(!resolved_override) {+ goto RETURN_EARLY;+ }+ }+ } else {+ i++;+ j++;+ }+ }++ RETURN_EARLY:++ if(!resolved_override || (exact_rule_match && query_tokens.size() != absorbed_tokens.size())) {+ delete [] override_ids;+ override_ids_len = 0;+ return false;+ }++ return resolved_override;+}++void Index::process_filter_overrides(const std::vector<const override_t*>& filter_overrides,+ std::vector<query_tokens_t>& field_query_tokens,+ token_ordering token_order,+ std::vector<filter>& filters,+ uint32_t** filter_ids,+ uint32_t& filter_ids_length) const {++ for(auto& override: filter_overrides) {+ if(!override->rule.dynamic_query) {+ // simple static filtering: add to filter_by and rewrite query if needed+ // we will cover both the original query and the synonym variants+ size_t orig_filters_size = filters.size();++ static_filter_query_eval(override, field_query_tokens[0].q_include_tokens, filters);+ for(auto& syn_tokens: field_query_tokens[0].q_synonyms) {+ static_filter_query_eval(override, syn_tokens, filters);+ }++ if(orig_filters_size != filters.size()) {+ // means that we have been able to resolve an override+ return ;+ }+ } else {+ // need to extract placeholder field names from the search query, filter on them and rewrite query+ // we will again cover both original query and synonyms++ std::vector<std::string> rule_parts;+ StringUtils::split(override->rule.query, rule_parts, " ");++ std::vector<std::string>& query_tokens = field_query_tokens[0].q_include_tokens;++ uint32_t* field_override_ids = nullptr;+ size_t field_override_ids_len = 0;+ bool exact_rule_match = override->rule.match == override_t::MATCH_EXACT;++ std::set<std::string> absorbed_tokens;+ bool resolved_override = resolve_override(rule_parts, exact_rule_match, query_tokens, token_order,+ absorbed_tokens, field_override_ids, field_override_ids_len);++ if(resolved_override && override->mutate_query_string) {+ std::vector<std::string> new_tokens;+ for(auto& token: query_tokens) {+ if(absorbed_tokens.count(token) == 0) {+ new_tokens.emplace_back(token);+ }+ }++ for(size_t j=0; j < field_query_tokens.size(); j++) {+ if(new_tokens.empty()) {+ field_query_tokens[j].q_include_tokens = {"*"};+ } else {+ field_query_tokens[j].q_include_tokens = new_tokens;+ }}}+ // now resolve synonyms+for(size_t i = 0; i < field_query_tokens[0].q_synonyms.size(); i++) {uint32_t* synonym_override_ids = nullptr;size_t synonym_override_ids_len = 0;+ absorbed_tokens.clear();- std::vector<std::string> syn_tokens = field_query_tokens[0].q_synonyms[i];- check_for_overrides(token_order, field_name, override->mutate_query_string,- synonym_override_ids, synonym_override_ids_len,syn_tokens);+ std::vector<std::string>& syn_tokens = field_query_tokens[0].q_synonyms[i];+ resolved_override = resolve_override(rule_parts, exact_rule_match, syn_tokens, token_order, absorbed_tokens,+ synonym_override_ids, synonym_override_ids_len);- for(size_t j=0; j < field_query_tokens.size(); j++) {- if(syn_tokens.empty()) {- field_query_tokens[j].q_synonyms[i] = {"*"};- } else {- field_query_tokens[j].q_synonyms[i] = syn_tokens;+ if(resolved_override && override->mutate_query_string) {+ std::vector<std::string> new_tokens;+ for (auto& token: syn_tokens) {+ if (absorbed_tokens.count(token) == 0) {+ new_tokens.emplace_back(token);+ }+ }++ for(size_t j=0; j < field_query_tokens.size(); j++) {+ if(new_tokens.empty()) {+ field_query_tokens[j].q_synonyms[i] = {"*"};+ } else {+ field_query_tokens[j].q_synonyms[i] = new_tokens;+ }}}+// result ids of synonyms will be OReduint32_t* syn_field_override_ids = nullptr;@@ -1575,31 +1714,31 @@ void Index::process_dynamic_overrides(const std::vector<const override_t*>& dynadelete [] field_override_ids;field_override_ids = syn_field_override_ids;}- }- if(filter_ids_length != 0) {- uint32_t* filtered_results = nullptr;- filter_ids_length = ArrayUtils::and_scalar(field_override_ids, field_override_ids_len, *filter_ids,- filter_ids_length, &filtered_results);- delete [] *filter_ids;- delete [] field_override_ids;- *filter_ids = filtered_results;- } else {- *filter_ids = field_override_ids;- filter_ids_length = field_override_ids_len;+ if(field_override_ids_len != 0) {+ if(filter_ids_length != 0) {+ uint32_t* filtered_results = nullptr;+ filter_ids_length = ArrayUtils::and_scalar(field_override_ids, field_override_ids_len, *filter_ids,+ filter_ids_length, &filtered_results);+ delete [] *filter_ids;+ delete [] field_override_ids;+ *filter_ids = filtered_results;+ } else {+ *filter_ids = field_override_ids;+ filter_ids_length = field_override_ids_len;+ }++ return ;+ }}}}-void-Index::check_for_overrides(const token_ordering& token_order, const string& field_name, bool mutate_query_string,- uint32_t*& field_override_ids,- size_t& field_override_ids_len, std::vector<std::string>& actual_tokens) const {-- std::vector<std::string> local_tokens = actual_tokens;- std::vector<std::string>& tokens = mutate_query_string ? actual_tokens : local_tokens;+bool Index::check_for_overrides2(const token_ordering& token_order, const string& field_name, const bool slide_window,+ uint32_t*& field_override_ids, size_t& field_override_ids_len,+ bool exact_rule_match, std::vector<std::string>& tokens,+ std::set<std::string>& absorbed_tokens) const {- DO_WINDOWING:for(size_t window_len = tokens.size(); window_len > 0; window_len--) {for(size_t start_index = 0; start_index+window_len-1 < tokens.size(); start_index++) {std::vector<token_t> window_tokens;@@ -1640,24 +1779,33 @@ Index::check_for_overrides(const token_ordering& token_order, const string& fielfield_override_ids_len = result_ids_len;}- // remove window_tokens from `tokens` and repeat search+ // remove window_tokens from `tokens`std::vector<std::string> new_tokens;- for(const auto& token: tokens) {+ for(size_t new_i = start_index; new_i < tokens.size(); new_i++) {+ const auto& token = tokens[new_i];if(window_tokens_set.count(token) == 0) {new_tokens.emplace_back(token);+ } else {+ absorbed_tokens.insert(token);}}tokens = new_tokens;- goto DO_WINDOWING;+ return true;+ }++ if(!slide_window) {+ break;}}}++ return false;}void Index::search(std::vector<query_tokens_t>& field_query_tokens,const std::vector<search_field_t>& search_fields,- const std::vector<filter>& filters,+ std::vector<filter>& filters,std::vector<facet>& facets, facet_query_t& facet_query,const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,const std::vector<uint32_t> & excluded_ids,@@ -1674,7 +1822,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,const size_t typo_tokens_threshold,const size_t group_limit,const std::vector<std::string>& group_by_fields,- const std::vector<const override_t*>& dynamic_overrides,+ const std::vector<const override_t*>& filter_overrides,const std::string& default_sorting_field,bool prioritize_exact_match,const bool exhaustive_search,@@ -1685,7 +1833,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,// process the filtersuint32_t* filter_ids = nullptr;- uint32_t filter_ids_length = do_filtering(&filter_ids, filters);+ uint32_t filter_ids_length = 0;std::shared_lock lock(mutex);@@ -1705,7 +1853,14 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,std::vector<uint32_t> curated_ids_sorted(curated_ids.begin(), curated_ids.end());std::sort(curated_ids_sorted.begin(), curated_ids_sorted.end());- process_dynamic_overrides(dynamic_overrides, field_query_tokens, token_order, &filter_ids, filter_ids_length);+ process_filter_overrides(filter_overrides, field_query_tokens, token_order, filters,+ &filter_ids, filter_ids_length);++ lock.unlock();++ do_filtering(filter_ids, filter_ids_length, filters);++ lock.lock();// Order of `fields` are used to sort results//auto begin = std::chrono::high_resolution_clock::now();diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex eff9f8a8..bd8c91fd 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -262,7 +262,7 @@ TEST_F(CollectionOverrideTest, OverrideJSONValidation) {parse_op = override_t::parse(include_json2, "", override2);ASSERT_FALSE(parse_op.ok());- ASSERT_STREQ("Must contain one of:`includes`, `excludes`, `dynamic_filters`.", parse_op.error().c_str());+ ASSERT_STREQ("Must contain one of:`includes`, `excludes`, `filter_by`.", parse_op.error().c_str());include_json2["includes"] = nlohmann::json::array();include_json2["includes"][0] = 100;@@ -817,36 +817,60 @@ TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {doc2["brand"] = "Adidas";doc2["points"] = 5;+ nlohmann::json doc3;+ doc3["id"] = "2";+ doc3["name"] = "Running Shoes";+ doc3["category"] = "sports";+ doc3["brand"] = "Nike";+ doc3["points"] = 5;+ASSERT_TRUE(coll1->add(doc1.dump()).ok());ASSERT_TRUE(coll1->add(doc2.dump()).ok());+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };auto results = coll1->search("shoes", {"name", "category", "brand"}, "",{}, sort_fields, {2, 2, 2}, 10).get();- ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ(3, results["hits"].size());ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("2", results["hits"][2]["document"]["id"].get<std::string>());// with override, results will be differentnlohmann::json override_json = {- {"id", "dynamic-filters"},+ {"id", "dynamic-cat-filter"},{"rule", {- {"query", ".*"},- {"match", override_t::MATCH_EXACT}+ {"query", "{category}"},+ {"match", override_t::MATCH_EXACT}}},{"mutate_query_string", true},- {"dynamic_filters", "category,brand"}+ {"filter_by", "category: {category}"}};override_t override;- auto op = override_t::parse(override_json, "dynamic-filters", override);+ auto op = override_t::parse(override_json, "dynamic-cat-filter", override);ASSERT_TRUE(op.ok());+ coll1->add_override(override);+ override_json = {+ {"id", "dynamic-brand-cat-filter"},+ {+ "rule", {+ {"query", "{brand} {category}"},+ {"match", override_t::MATCH_EXACT}+ }+ },+ {"mutate_query_string", true},+ {"filter_by", "category: {category} && brand: {brand}"}+ };++ op = override_t::parse(override_json, "dynamic-brand-cat-filter", override);+ ASSERT_TRUE(op.ok());coll1->add_override(override);results = coll1->search("shoes", {"name", "category", "brand"}, "",@@ -856,6 +880,12 @@ TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());+ // should not apply filter for non-exact case+ results = coll1->search("running shoes", {"name", "category", "brand"}, "",+ {}, sort_fields, {2, 2, 2}, 10).get();++ ASSERT_EQ(3, results["hits"].size());+results = coll1->search("adidas shoes", {"name", "category", "brand"}, "",{}, sort_fields, {2, 2, 2}, 10).get();@@ -868,35 +898,185 @@ TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {{"id", "dynamic-filters-bad1"},{"rule", {- {"query", ".*"},- {"match", override_t::MATCH_EXACT}+ {"query", "{brand}"},+ {"match", override_t::MATCH_EXACT}}},{"mutate_query_string", true},- {"dynamic_filters", ""}+ {"filter_by", ""}};override_t override_bad1;op = override_t::parse(override_json_bad1, "dynamic-filters-bad1", override_bad1);ASSERT_FALSE(op.ok());- ASSERT_EQ("The `dynamic_filters` must be a comma separated string of filter field names.", op.error());+ ASSERT_EQ("The `filter_by` must be a non-empty string.", op.error());nlohmann::json override_json_bad2 = {{"id", "dynamic-filters-bad2"},{"rule", {- {"query", ".*"},+ {"query", "{brand}"},{"match", override_t::MATCH_EXACT}}},{"mutate_query_string", true},- {"dynamic_filters", {"foo", "bar"}}+ {"filter_by", {"foo", "bar"}}};override_t override_bad2;op = override_t::parse(override_json_bad2, "dynamic-filters-bad2", override_bad2);ASSERT_FALSE(op.ok());- ASSERT_EQ("The `dynamic_filters` must be a comma separated string of filter field names.", op.error());+ ASSERT_EQ("The `filter_by` must be a string.", op.error());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionOverrideTest, DynamicFilteringMultiplePlaceholders) {+ Collection* coll1;++ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("category", field_types::STRING, true),+ field("brand", field_types::STRING, true),+ field("color", field_types::STRING, true),+ field("points", field_types::INT32, false)};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Retro Shoes";+ doc1["category"] = "shoes";+ doc1["color"] = "yellow";+ doc1["brand"] = "Nike Air Jordan";+ doc1["points"] = 3;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "Baseball";+ doc2["category"] = "shoes";+ doc2["color"] = "white";+ doc2["brand"] = "Adidas";+ doc2["points"] = 5;++ nlohmann::json doc3;+ doc3["id"] = "2";+ doc3["name"] = "Running Shoes";+ doc3["category"] = "sports";+ doc3["color"] = "grey";+ doc3["brand"] = "Nike";+ doc3["points"] = 5;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());++ std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC"), sort_by("points", "DESC")};++ nlohmann::json override_json = {+ {"id", "dynamic-cat-filter"},+ {+ "rule", {+ {"query", "{brand} {color} shoes"},+ {"match", override_t::MATCH_CONTAINS}+ }+ },+ {"mutate_query_string", true},+ {"filter_by", "brand: {brand} && color: {color}"}+ };++ override_t override;+ auto op = override_t::parse(override_json, "dynamic-cat-filter", override);+ ASSERT_TRUE(op.ok());+ coll1->add_override(override);++ // not an exact match of rule (because of "light") so all results will be fetched, not just Air Jordan brand+ auto results = coll1->search("Nike Air Jordan light yellow shoes", {"name", "category", "brand"}, "",+ {}, sort_fields, {2, 2, 2}, 10).get();++ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());++ // query with tokens at the start that preceding the placeholders in the rule+ results = coll1->search("New Nike Air Jordan yellow shoes", {"name", "category", "brand"}, "",+ {}, sort_fields, {2, 2, 2}, 10).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionOverrideTest, DynamicFilteringTokensBetweenPlaceholders) {+ Collection* coll1;++ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("category", field_types::STRING, true),+ field("brand", field_types::STRING, true),+ field("color", field_types::STRING, true),+ field("points", field_types::INT32, false)};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Retro Shoes";+ doc1["category"] = "shoes";+ doc1["color"] = "yellow";+ doc1["brand"] = "Nike Air Jordan";+ doc1["points"] = 3;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "Baseball";+ doc2["category"] = "shoes";+ doc2["color"] = "white";+ doc2["brand"] = "Adidas";+ doc2["points"] = 5;++ nlohmann::json doc3;+ doc3["id"] = "2";+ doc3["name"] = "Running Shoes";+ doc3["category"] = "sports";+ doc3["color"] = "grey";+ doc3["brand"] = "Nike";+ doc3["points"] = 5;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());++ std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC"), sort_by("points", "DESC")};++ nlohmann::json override_json = {+ {"id", "dynamic-cat-filter"},+ {+ "rule", {+ {"query", "{brand} shoes {color}"},+ {"match", override_t::MATCH_CONTAINS}+ }+ },+ {"mutate_query_string", true},+ {"filter_by", "brand: {brand} && color: {color}"}+ };++ override_t override;+ auto op = override_t::parse(override_json, "dynamic-cat-filter", override);+ ASSERT_TRUE(op.ok());+ coll1->add_override(override);++ auto results = coll1->search("Nike Air Jordan shoes yellow", {"name", "category", "brand"}, "",+ {}, sort_fields, {2, 2, 2}, 10).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());collectionManager.drop_collection("coll1");}@@ -952,12 +1132,12 @@ TEST_F(CollectionOverrideTest, DynamicFilteringWithSynonyms) {{"id", "dynamic-filters"},{"rule", {- {"query", ".*"},+ {"query", "{category}"},{"match", override_t::MATCH_EXACT}}},{"mutate_query_string", true},- {"dynamic_filters", "category,brand"}+ {"filter_by", "category: {category}"}};override_t override;@@ -1025,7 +1205,7 @@ TEST_F(CollectionOverrideTest, StaticFiltering) {}},{"mutate_query_string", true},- {"dynamic_filters", "price:> 100"}+ {"filter_by", "price:> 100"}};override_t override_contains;@@ -1043,7 +1223,7 @@ TEST_F(CollectionOverrideTest, StaticFiltering) {}},{"mutate_query_string", true},- {"dynamic_filters", "price:< 100"}+ {"filter_by", "price:< 100"}};override_t override_exact;@@ -1127,7 +1307,7 @@ TEST_F(CollectionOverrideTest, FilteringWithAndWithoutQueryStringMutation) {}},{"mutate_query_string", false},- {"dynamic_filters", "price:> 200"}+ {"filter_by", "price:> 200"}};override_t override_contains;@@ -1155,7 +1335,7 @@ TEST_F(CollectionOverrideTest, FilteringWithAndWithoutQueryStringMutation) {}},{"mutate_query_string", true},- {"dynamic_filters", "price:> 200"}+ {"filter_by", "price:> 200"}};op = override_t::parse(override_json_contains, "static-filters", override_contains);diff --git a/test/index_test.cpp b/test/index_test.cppindex 8ca0ef07..b06e8ae8 100644--- a/test/index_test.cpp+++ b/test/index_test.cpp@@ -12,7 +12,7 @@ TEST(IndexTest, ScrubReindexDoc) {ThreadPool pool(4);- Index index("index", &pool, search_schema, {}, {}, {}, {});+ Index index("index", 1, nullptr, &pool, search_schema, {}, {}, {}, {});nlohmann::json old_doc;old_doc["id"] = "1";old_doc["title"] = "One more thing.";