Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Dynamic filtering using field interpolation.

Kishore Nallan70059dec15c6
diff --git a/include/collection.h b/include/collection.h
index b1697f2b..2b3baba4 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -207,7 +207,7 @@ private:
const std::map<size_t, std::vector<std::string>>& pinned_hits,
const std::vector<std::string>& hidden_hits,
std::map<size_t, std::vector<uint32_t>>& include_ids,
- std::vector<uint32_t>& excluded_ids, std::vector<const override_t*>& dynamic_overrides,
+ std::vector<uint32_t>& excluded_ids, std::vector<const override_t*>& filter_overrides,
std::vector<filter>& filters) const;
Option<bool> check_and_update_schema(nlohmann::json& document, const DIRTY_VALUES& dirty_values);
@@ -228,13 +228,6 @@ private:
return std::tie(a_count, a_value_size) > std::tie(b_count, b_value_size);
}
- Option<bool> parse_filter_query(const std::string& simple_filter_query, std::vector<filter>& filters) const;
-
- static Option<bool> parse_geopoint_filter_value(std::string& raw_value,
- const std::string& format_err_msg,
- std::string& processed_filter_val,
- NUM_COMPARATOR& num_comparator);
-
static Option<bool> parse_pinned_hits(const std::string& pinned_hits_str,
std::map<size_t, std::vector<std::string>>& pinned_hits);
diff --git a/include/field.h b/include/field.h
index a36807d9..756b6692 100644
--- a/include/field.h
+++ b/include/field.h
@@ -6,6 +6,7 @@
#include "option.h"
#include "string_utils.h"
#include "logger.h"
+#include <sparsepp.h>
#include "json.hpp"
namespace field_types {
@@ -463,6 +464,17 @@ struct filter {
return Option<NUM_COMPARATOR>(num_comparator);
}
+
+ static Option<bool> parse_geopoint_filter_value(std::string& raw_value,
+ const std::string& format_err_msg,
+ std::string& processed_filter_val,
+ NUM_COMPARATOR& num_comparator);
+
+ static Option<bool> parse_filter_query(const std::string& simple_filter_query,
+ const std::unordered_map<std::string, field>& search_schema,
+ Store* store,
+ const std::string& doc_id_prefix,
+ std::vector<filter>& filters);
};
namespace sort_field_const {
diff --git a/include/index.h b/include/index.h
index 8f2453b1..e8edf62a 100644
--- a/include/index.h
+++ b/include/index.h
@@ -54,6 +54,7 @@ struct override_t {
struct rule_t {
std::string query;
std::string match;
+ bool dynamic_query = false;
};
struct add_hit_t {
@@ -71,7 +72,7 @@ struct override_t {
std::vector<add_hit_t> add_hits;
std::vector<drop_hit_t> drop_hits;
- std::vector<std::string> dynamic_filters;
+ std::string filter_by;
bool mutate_query_string = false;
override_t() {}
@@ -90,8 +91,8 @@ struct override_t {
}
if(override_json.count("includes") == 0 && override_json.count("excludes") == 0 &&
- override_json.count("dynamic_filters") == 0) {
- return Option<bool>(400, "Must contain one of:`includes`, `excludes`, `dynamic_filters`.");
+ override_json.count("filter_by") == 0) {
+ return Option<bool>(400, "Must contain one of:`includes`, `excludes`, `filter_by`.");
}
if(override_json.count("includes") != 0) {
@@ -139,15 +140,13 @@ struct override_t {
}
- if(override_json.count("dynamic_filters") != 0) {
- if(!override_json["dynamic_filters"].is_string()) {
- return Option<bool>(400, "The `dynamic_filters` must be "
- "a comma separated string of filter field names.");
+ if(override_json.count("filter_by") != 0) {
+ if(!override_json["filter_by"].is_string()) {
+ return Option<bool>(400, "The `filter_by` must be a string.");
}
- if(override_json["dynamic_filters"].get<std::string>().empty()) {
- return Option<bool>(400, "The `dynamic_filters` must be "
- "a comma separated string of filter field names.");
+ if(override_json["filter_by"].get<std::string>().empty()) {
+ return Option<bool>(400, "The `filter_by` must be a non-empty string.");
}
}
@@ -187,10 +186,25 @@ struct override_t {
}
}
- if (override_json.count("dynamic_filters") != 0) {
- std::vector<std::string> filter_fields;
- StringUtils::split(override_json["dynamic_filters"].get<std::string>(), filter_fields, ",");
- override.dynamic_filters = filter_fields;
+ if (override_json.count("filter_by") != 0) {
+ override.filter_by = override_json["filter_by"].get<std::string>();
+ }
+
+ // we have to also detect if it is a dynamic query rule
+ size_t i = 0;
+ while(i < override.rule.query.size()) {
+ if(override.rule.query[i] == '{') {
+ // look for closing curly
+ i++;
+ while(i < override.rule.query.size()) {
+ if(override.rule.query[i] == '}') {
+ override.rule.dynamic_query = true;
+ break;
+ }
+ i++;
+ }
+ }
+ i++;
}
return Option<bool>(true);
@@ -246,7 +260,7 @@ struct search_args {
size_t all_result_ids_len;
bool exhaustive_search;
size_t concurrency;
- const std::vector<const override_t*>& dynamic_overrides;
+ const std::vector<const override_t*>& filter_overrides;
spp::sparse_hash_set<uint64_t> groups_processed;
std::vector<std::vector<art_leaf*>> searched_queries;
@@ -278,7 +292,7 @@ struct search_args {
group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),
prioritize_exact_match(prioritize_exact_match), all_result_ids_len(0),
exhaustive_search(exhaustive_search), concurrency(concurrency),
- dynamic_overrides(dynamic_overrides) {
+ filter_overrides(dynamic_overrides) {
const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatory
topster = new Topster(topster_size, group_limit);
@@ -347,6 +361,10 @@ private:
std::string name;
+ const uint32_t collection_id;
+
+ Store* store;
+
size_t num_documents;
std::unordered_map<std::string, field> search_schema;
@@ -397,12 +415,26 @@ private:
size_t group_limit, const std::vector<std::string>& group_by_fields,
const uint32_t* result_ids, size_t results_size) const;
- void process_dynamic_overrides(const std::vector<const override_t*>& dynamic_overrides,
+ void static_filter_query_eval(const override_t* override, std::vector<std::string>& tokens,
+ std::vector<filter>& filters) const;
+
+ void process_filter_overrides(const std::vector<const override_t*>& filter_overrides,
std::vector<query_tokens_t>& field_query_tokens,
- const token_ordering token_order,
+ token_ordering token_order,
+ std::vector<filter>& filters,
uint32_t** filter_ids,
uint32_t& filter_ids_length) const;
+ bool resolve_override(const std::vector<std::string>& rule_parts, const bool exact_rule_match,
+ const std::vector<std::string>& query_tokens,
+ token_ordering token_order, std::set<std::string>& absorbed_tokens,
+ uint32_t*& override_ids, size_t& override_ids_len) const;
+
+ bool check_for_overrides2(const token_ordering& token_order, const string& field_name, const bool slide_window,
+ uint32_t*& field_override_ids, size_t& field_override_ids_len,
+ bool exact_rule_match, std::vector<std::string>& tokens,
+ std::set<std::string>& absorbed_tokens) const;
+
void search_field(const uint8_t & field_id,
std::vector<token_t>& query_tokens,
std::vector<token_t>& search_tokens,
@@ -510,6 +542,8 @@ public:
Index() = delete;
Index(const std::string& name,
+ const uint32_t collection_id,
+ const Store* store,
ThreadPool* thread_pool,
const std::unordered_map<std::string, field>& search_schema,
const std::map<std::string, field>& facet_schema,
@@ -563,7 +597,7 @@ public:
void search(std::vector<query_tokens_t>& field_query_tokens,
const std::vector<search_field_t> & search_fields,
- const std::vector<filter> & filters, std::vector<facet> & facets,
+ std::vector<filter> & filters, std::vector<facet> & facets,
facet_query_t & facet_query,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
const std::vector<uint32_t> & excluded_ids,
@@ -579,7 +613,7 @@ public:
const size_t typo_tokens_threshold,
const size_t group_limit,
const std::vector<std::string>& group_by_fields,
- const std::vector<const override_t*>& dynamic_overrides,
+ const std::vector<const override_t*>& filter_overrides,
const std::string& default_sorting_field,
bool prioritize_exact_match,
bool exhaustive_search,
@@ -606,7 +640,7 @@ public:
art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);
- uint32_t do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) const;
+ void do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length, const std::vector<filter>& filters) const;
void refresh_schemas(const std::vector<field>& new_fields);
@@ -622,10 +656,6 @@ public:
const std::string& fallback_field_type,
const DIRTY_VALUES& dirty_values);
- void check_for_overrides(const token_ordering& token_order, const string& field_name, bool mutate_query_string,
- uint32_t*& field_override_ids,
- size_t& field_override_ids_len, std::vector<std::string>& tokens) const;
-
void search_wildcard(const std::vector<std::string>& qtokens, const std::vector<filter>& filters,
const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
const std::vector<sort_by>& sort_fields_std, Topster* topster, Topster* curated_topster,
diff --git a/src/collection.cpp b/src/collection.cpp
index f277c5d3..19bc6638 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -377,7 +377,7 @@ void Collection::curate_results(string& actual_query, bool enable_overrides, boo
const std::map<size_t, std::vector<std::string>>& pinned_hits,
const std::vector<std::string>& hidden_hits,
std::map<size_t, std::vector<uint32_t>>& include_ids,
- std::vector<uint32_t>& excluded_ids, std::vector<const override_t*>& dynamic_overrides,
+ std::vector<uint32_t>& excluded_ids, std::vector<const override_t*>& filter_overrides,
std::vector<filter>& filters) const {
std::set<uint32_t> excluded_set;
@@ -403,9 +403,9 @@ void Collection::curate_results(string& actual_query, bool enable_overrides, boo
for(const auto& override_kv: overrides) {
const auto& override = override_kv.second;
- // static overrides are applied first as they precedence over dynamic overrides
- if(!override.dynamic_filters.empty() && override.rule.query == ".*") {
- dynamic_overrides.push_back(&override);
+ // ID-based overrides are applied first as they take precedence over filter-based overrides
+ if(!override.filter_by.empty()) {
+ filter_overrides.push_back(&override);
continue;
}
@@ -432,21 +432,6 @@ void Collection::curate_results(string& actual_query, bool enable_overrides, boo
include_ids[hit.position].push_back(seq_id);
}
}
-
- for(const auto& filter_str: override.dynamic_filters) {
- auto op = parse_filter_query(filter_str, filters);
- if(op.ok() && override.mutate_query_string) {
- actual_query.replace(
- actual_query.find(override.rule.query),
- override.rule.query.size(),
- ""
- );
-
- if(StringUtils::trim(actual_query).empty()) {
- actual_query = "*";
- }
- }
- }
}
}
}
@@ -587,8 +572,10 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
}
}
+ const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";
std::vector<filter> filters;
- Option<bool> parse_filter_op = parse_filter_query(simple_filter_query, filters);
+ Option<bool> parse_filter_op = filter::parse_filter_query(simple_filter_query, search_schema,
+ store, doc_id_prefix, filters);
if(!parse_filter_op.ok()) {
return Option<nlohmann::json>(parse_filter_op.code(), parse_filter_op.error());
}
@@ -840,10 +827,10 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
std::vector<std::string> hidden_hits;
StringUtils::split(hidden_hits_str, hidden_hits, ",");
- std::vector<const override_t*> dynamic_overrides;
+ std::vector<const override_t*> filter_overrides;
std::string query = raw_query;
curate_results(query, enable_overrides, pre_segmented_query, pinned_hits, hidden_hits,
- include_ids, excluded_ids, dynamic_overrides, filters);
+ include_ids, excluded_ids, filter_overrides, filters);
/*for(auto& kv: include_ids) {
LOG(INFO) << "key: " << kv.first;
@@ -911,12 +898,12 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
size_t index_id = 0;
search_args* search_params = new search_args(field_query_tokens, weighted_search_fields,
- filters, facets, included_ids, excluded_ids,
- sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
- per_page, page, token_order, prefixes,
- drop_tokens_threshold, typo_tokens_threshold,
- group_by_fields, group_limit, default_sorting_field, prioritize_exact_match,
- exhaustive_search, 4, dynamic_overrides);
+ filters, facets, included_ids, excluded_ids,
+ sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
+ per_page, page, token_order, prefixes,
+ drop_tokens_threshold, typo_tokens_threshold,
+ group_by_fields, group_limit, default_sorting_field, prioritize_exact_match,
+ exhaustive_search, 4, filter_overrides);
index->run_search(search_params);
@@ -1361,15 +1348,18 @@ Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,
std::vector<std::pair<size_t, uint32_t*>>& index_ids) {
std::shared_lock lock(mutex);
+ const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";
std::vector<filter> filters;
- Option<bool> filter_op = parse_filter_query(simple_filter_query, filters);
+ Option<bool> filter_op = filter::parse_filter_query(simple_filter_query, search_schema,
+ store, doc_id_prefix, filters);
if(!filter_op.ok()) {
return filter_op;
}
uint32_t* filter_ids = nullptr;
- size_t filter_ids_len = index->do_filtering(&filter_ids, filters);
+ uint32_t filter_ids_len = 0;
+ index->do_filtering(filter_ids, filter_ids_len, filters);
index_ids.emplace_back(filter_ids_len, filter_ids);
return Option<bool>(true);
@@ -2025,346 +2015,6 @@ const Index* Collection::_get_index() const {
return index;
}
-Option<bool> Collection::parse_geopoint_filter_value(std::string& raw_value,
- const std::string& format_err_msg,
- std::string& processed_filter_val,
- NUM_COMPARATOR& num_comparator) {
-
- num_comparator = LESS_THAN_EQUALS;
-
- if(!(raw_value[0] == '(' && raw_value[raw_value.size() - 1] == ')')) {
- return Option<bool>(400, format_err_msg);
- }
-
- std::vector<std::string> filter_values;
- auto raw_val_without_paran = raw_value.substr(1, raw_value.size() - 2);
- StringUtils::split(raw_val_without_paran, filter_values, ",");
-
- // we will end up with: "10.45 34.56 2 km" or a geo polygon
-
- if(filter_values.size() < 3) {
- return Option<bool>(400, format_err_msg);
- }
-
- // do validation: format should match either a point + radius or polygon
-
- size_t num_floats = 0;
- for(const auto& fvalue: filter_values) {
- if(StringUtils::is_float(fvalue)) {
- num_floats++;
- }
- }
-
- bool is_polygon = (num_floats == filter_values.size());
- if(!is_polygon) {
- // we have to ensure that this is a point + radius match
- if(!StringUtils::is_float(filter_values[0]) || !StringUtils::is_float(filter_values[1])) {
- return Option<bool>(400, format_err_msg);
- }
- }
-
- if(is_polygon) {
- processed_filter_val = raw_val_without_paran;
- } else {
- // point + radius
- std::vector<std::string> dist_values;
- StringUtils::split(filter_values[2], dist_values, " ");
-
- if(dist_values.size() != 2) {
- return Option<bool>(400, format_err_msg);
- }
-
- if(dist_values[1] != "km" && dist_values[1] != "mi") {
- return Option<bool>(400, "Unit must be either `km` or `mi`.");
- }
-
- processed_filter_val = filter_values[0] + ", " + filter_values[1] + ", " + // co-ords
- dist_values[0] + ", " + dist_values[1]; // X km
- }
-
- return Option<bool>(true);
-}
-
-Option<bool> Collection::parse_filter_query(const std::string& simple_filter_query,
- std::vector<filter>& filters) const {
-
- std::vector<filter> exclude_filters; // to ensure that they go last in the list of filters
-
- std::vector<std::string> filter_blocks;
- StringUtils::split(simple_filter_query, filter_blocks, "&&");
-
- for(const std::string & filter_block: filter_blocks) {
- // split into [field_name, value]
- size_t found_index = filter_block.find(':');
-
- if(found_index == std::string::npos) {
- return Option<bool>(400, "Could not parse the filter query.");
- }
-
- std::string&& field_name = filter_block.substr(0, found_index);
- StringUtils::trim(field_name);
-
- if(field_name == "id") {
- filter id_filter;
- std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);
- StringUtils::trim(raw_value);
- id_filter = {field_name, {}, {}};
-
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> doc_ids;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), doc_ids, ",");
-
- for(std::string& doc_id: doc_ids) {
- // we have to convert the doc_id to seq id
- std::string seq_id_str;
- StoreStatus seq_id_status = store->get(get_doc_id_key(doc_id), seq_id_str);
-
- if(seq_id_status != StoreStatus::FOUND) {
- continue;
- }
-
- id_filter.values.push_back(seq_id_str);
- id_filter.comparators.push_back(EQUALS);
- }
- } else {
- std::string seq_id_str;
- StoreStatus seq_id_status = store->get(get_doc_id_key(raw_value), seq_id_str);
- if(seq_id_status == StoreStatus::FOUND) {
- id_filter.values.push_back(seq_id_str);
- id_filter.comparators.push_back(EQUALS);
- }
- }
-
- if(!id_filter.values.empty()) {
- filters.push_back(id_filter);
- }
-
- continue;
- }
-
- if(search_schema.count(field_name) == 0) {
- return Option<bool>(404, "Could not find a filter field named `" + field_name + "` in the schema.");
- }
-
- field _field = search_schema.at(field_name);
- std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);
- StringUtils::trim(raw_value);
- filter f;
-
- // skip past optional `:=` operator, which has no meaning for non-string fields
- if(!_field.is_string() && raw_value[0] == '=') {
- size_t filter_value_index = 0;
- while(raw_value[++filter_value_index] == ' ');
- raw_value = raw_value.substr(filter_value_index);
- }
-
- if(_field.is_integer() || _field.is_float()) {
- // could be a single value or a list
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
-
- f = {field_name, {}, {}};
-
- for(std::string & filter_value: filter_values) {
- Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(filter_value);
- if(!op_comparator.ok()) {
- return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());
- }
-
- if(op_comparator.get() == RANGE_INCLUSIVE) {
- // split the value around range operator to extract bounds
- std::vector<std::string> range_values;
- StringUtils::split(filter_value, range_values, filter::RANGE_OPERATOR());
- for(const std::string& range_value: range_values) {
- auto validate_op = filter::validate_numerical_filter_value(_field, range_value);
- if(!validate_op.ok()) {
- return validate_op;
- }
-
- f.values.push_back(range_value);
- f.comparators.push_back(op_comparator.get());
- }
- } else {
- auto validate_op = filter::validate_numerical_filter_value(_field, filter_value);
- if(!validate_op.ok()) {
- return validate_op;
- }
-
- f.values.push_back(filter_value);
- f.comparators.push_back(op_comparator.get());
- }
- }
-
- } else {
- Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);
- if(!op_comparator.ok()) {
- return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());
- }
-
- if(op_comparator.get() == RANGE_INCLUSIVE) {
- // split the value around range operator to extract bounds
- std::vector<std::string> range_values;
- StringUtils::split(raw_value, range_values, filter::RANGE_OPERATOR());
-
- f.field_name = field_name;
- for(const std::string& range_value: range_values) {
- auto validate_op = filter::validate_numerical_filter_value(_field, range_value);
- if(!validate_op.ok()) {
- return validate_op;
- }
-
- f.values.push_back(range_value);
- f.comparators.push_back(op_comparator.get());
- }
- } else {
- auto validate_op = filter::validate_numerical_filter_value(_field, raw_value);
- if(!validate_op.ok()) {
- return validate_op;
- }
- f = {field_name, {raw_value}, {op_comparator.get()}};
- }
- }
- } else if(_field.is_bool()) {
- NUM_COMPARATOR bool_comparator = EQUALS;
- size_t filter_value_index = 0;
-
- if(raw_value[0] == '=') {
- bool_comparator = EQUALS;
- while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
- } else if(raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
- bool_comparator = NOT_EQUALS;
- filter_value_index++;
- while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
- }
-
- if(filter_value_index != 0) {
- raw_value = raw_value.substr(filter_value_index);
- }
-
- if(filter_value_index == raw_value.size()) {
- return Option<bool>(400, "Error with filter field `" + _field.name +
- "`: Filter value cannot be empty.");
- }
-
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
- f = {field_name, {}, {}};
-
- for(std::string & filter_value: filter_values) {
- if(filter_value != "true" && filter_value != "false") {
- return Option<bool>(400, "Values of filter field `" + _field.name +
- "`: must be `true` or `false`.");
- }
-
- filter_value = (filter_value == "true") ? "1" : "0";
- f.values.push_back(filter_value);
- f.comparators.push_back(bool_comparator);
- }
- } else {
- if(raw_value != "true" && raw_value != "false") {
- return Option<bool>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");
- }
-
- std::string bool_value = (raw_value == "true") ? "1" : "0";
- f = {field_name, {bool_value}, {bool_comparator}};
- }
-
- } else if(_field.is_geopoint()) {
- f = {field_name, {}, {}};
-
- const std::string& format_err_msg = "Value of filter field `" + _field.name +
- "`: must be in the `(-44.50, 170.29, 0.75 km)` or "
- "(56.33, -65.97, 23.82, -127.82) format.";
-
- NUM_COMPARATOR num_comparator;
-
- // could be a single value or a list
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, "),");
-
- for(std::string& filter_value: filter_values) {
- filter_value += ")";
- std::string processed_filter_val;
- auto parse_op = parse_geopoint_filter_value(filter_value, format_err_msg, processed_filter_val, num_comparator);
-
- if(!parse_op.ok()) {
- return parse_op;
- }
-
- f.values.push_back(processed_filter_val);
- f.comparators.push_back(num_comparator);
- }
- } else {
- // single value, e.g. (10.45, 34.56, 2 km)
- std::string processed_filter_val;
- auto parse_op = parse_geopoint_filter_value(raw_value, format_err_msg, processed_filter_val, num_comparator);
-
- if(!parse_op.ok()) {
- return parse_op;
- }
-
- f.values.push_back(processed_filter_val);
- f.comparators.push_back(num_comparator);
- }
-
- } else if(_field.is_string()) {
- size_t filter_value_index = 0;
- NUM_COMPARATOR str_comparator = CONTAINS;
-
- if(raw_value[0] == '=') {
- if(!_field.facet) {
- // EQUALS filtering on string is possible only on facet fields
- return Option<bool>(400, "To perform exact filtering, filter field `" +
- _field.name + "` must be a facet field.");
- }
-
- // string filter should be evaluated in strict "equals" mode
- str_comparator = EQUALS;
- while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
- } else if(raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
- if(!_field.facet) {
- // EXCLUDE filtering on string is possible only on facet fields
- return Option<bool>(400, "To perform exclude filtering, filter field `" +
- _field.name + "` must be a facet field.");
- }
-
- str_comparator = NOT_EQUALS;
- filter_value_index++;
- while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
- }
-
- if(filter_value_index == raw_value.size()) {
- return Option<bool>(400, "Error with filter field `" + _field.name +
- "`: Filter value cannot be empty.");
- }
-
- if(raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split_to_values(raw_value.substr(filter_value_index+1, raw_value.size() - filter_value_index - 2), filter_values);
- f = {field_name, filter_values, {str_comparator}};
- } else {
- f = {field_name, {raw_value.substr(filter_value_index)}, {str_comparator}};
- }
- } else {
- return Option<bool>(400, "Error with filter field `" + _field.name +
- "`: Unidentified field data type, see docs for supported data types.");
- }
-
- if(f.comparators.size() > 0 && f.comparators.front() == NOT_EQUALS) {
- exclude_filters.push_back(f);
- } else {
- filters.push_back(f);
- }
- }
-
- filters.insert( filters.end(), exclude_filters.begin(), exclude_filters.end() );
-
- return Option<bool>(true);
-}
-
Option<bool> Collection::parse_pinned_hits(const std::string& pinned_hits_str,
std::map<size_t, std::vector<std::string>>& pinned_hits) {
if(!pinned_hits_str.empty()) {
@@ -2721,6 +2371,8 @@ Index* Collection::init_index() {
}
return new Index(name+std::to_string(0),
+ collection_id,
+ store,
CollectionManager::get_instance().get_thread_pool(),
search_schema, facet_schema, sort_schema,
symbols_to_index, token_separators);
diff --git a/src/field.cpp b/src/field.cpp
new file mode 100644
index 00000000..b0ba565e
--- /dev/null
+++ b/src/field.cpp
@@ -0,0 +1,345 @@
+#include <store.h>
+#include "field.h"
+
+Option<bool> filter::parse_geopoint_filter_value(std::string& raw_value,
+ const std::string& format_err_msg,
+ std::string& processed_filter_val,
+ NUM_COMPARATOR& num_comparator) {
+
+ num_comparator = LESS_THAN_EQUALS;
+
+ if(!(raw_value[0] == '(' && raw_value[raw_value.size() - 1] == ')')) {
+ return Option<bool>(400, format_err_msg);
+ }
+
+ std::vector<std::string> filter_values;
+ auto raw_val_without_paran = raw_value.substr(1, raw_value.size() - 2);
+ StringUtils::split(raw_val_without_paran, filter_values, ",");
+
+ // we will end up with: "10.45 34.56 2 km" or a geo polygon
+
+ if(filter_values.size() < 3) {
+ return Option<bool>(400, format_err_msg);
+ }
+
+ // do validation: format should match either a point + radius or polygon
+
+ size_t num_floats = 0;
+ for(const auto& fvalue: filter_values) {
+ if(StringUtils::is_float(fvalue)) {
+ num_floats++;
+ }
+ }
+
+ bool is_polygon = (num_floats == filter_values.size());
+ if(!is_polygon) {
+ // we have to ensure that this is a point + radius match
+ if(!StringUtils::is_float(filter_values[0]) || !StringUtils::is_float(filter_values[1])) {
+ return Option<bool>(400, format_err_msg);
+ }
+ }
+
+ if(is_polygon) {
+ processed_filter_val = raw_val_without_paran;
+ } else {
+ // point + radius
+ std::vector<std::string> dist_values;
+ StringUtils::split(filter_values[2], dist_values, " ");
+
+ if(dist_values.size() != 2) {
+ return Option<bool>(400, format_err_msg);
+ }
+
+ if(dist_values[1] != "km" && dist_values[1] != "mi") {
+ return Option<bool>(400, "Unit must be either `km` or `mi`.");
+ }
+
+ processed_filter_val = filter_values[0] + ", " + filter_values[1] + ", " + // co-ords
+ dist_values[0] + ", " + dist_values[1]; // X km
+ }
+
+ return Option<bool>(true);
+}
+
+Option<bool> filter::parse_filter_query(const string& simple_filter_query,
+ const std::unordered_map<std::string, field>& search_schema,
+ Store* store,
+ const std::string& doc_id_prefix,
+ std::vector<filter>& filters) {
+
+ std::vector<filter> exclude_filters; // to ensure that they go last in the list of filters
+
+ std::vector<std::string> filter_blocks;
+ StringUtils::split(simple_filter_query, filter_blocks, "&&");
+
+ for(const std::string & filter_block: filter_blocks) {
+ // split into [field_name, value]
+ size_t found_index = filter_block.find(':');
+
+ if(found_index == std::string::npos) {
+ return Option<bool>(400, "Could not parse the filter query.");
+ }
+
+ std::string&& field_name = filter_block.substr(0, found_index);
+ StringUtils::trim(field_name);
+
+ if(field_name == "id") {
+ filter id_filter;
+ std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);
+ StringUtils::trim(raw_value);
+ id_filter = {field_name, {}, {}};
+
+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> doc_ids;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), doc_ids, ",");
+
+ for(std::string& doc_id: doc_ids) {
+ // we have to convert the doc_id to seq id
+ std::string seq_id_str;
+ StoreStatus seq_id_status = store->get(doc_id_prefix + doc_id, seq_id_str);
+
+ if(seq_id_status != StoreStatus::FOUND) {
+ continue;
+ }
+
+ id_filter.values.push_back(seq_id_str);
+ id_filter.comparators.push_back(EQUALS);
+ }
+ } else {
+ std::string seq_id_str;
+ StoreStatus seq_id_status = store->get(doc_id_prefix + raw_value, seq_id_str);
+ if(seq_id_status == StoreStatus::FOUND) {
+ id_filter.values.push_back(seq_id_str);
+ id_filter.comparators.push_back(EQUALS);
+ }
+ }
+
+ if(!id_filter.values.empty()) {
+ filters.push_back(id_filter);
+ }
+
+ continue;
+ }
+
+ if(search_schema.count(field_name) == 0) {
+ return Option<bool>(404, "Could not find a filter field named `" + field_name + "` in the schema.");
+ }
+
+ field _field = search_schema.at(field_name);
+ std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);
+ StringUtils::trim(raw_value);
+ filter f;
+
+ // skip past optional `:=` operator, which has no meaning for non-string fields
+ if(!_field.is_string() && raw_value[0] == '=') {
+ size_t filter_value_index = 0;
+ while(raw_value[++filter_value_index] == ' ');
+ raw_value = raw_value.substr(filter_value_index);
+ }
+
+ if(_field.is_integer() || _field.is_float()) {
+ // could be a single value or a list
+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+
+ f = {field_name, {}, {}};
+
+ for(std::string & filter_value: filter_values) {
+ Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(filter_value);
+ if(!op_comparator.ok()) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());
+ }
+
+ if(op_comparator.get() == RANGE_INCLUSIVE) {
+ // split the value around range operator to extract bounds
+ std::vector<std::string> range_values;
+ StringUtils::split(filter_value, range_values, filter::RANGE_OPERATOR());
+ for(const std::string& range_value: range_values) {
+ auto validate_op = filter::validate_numerical_filter_value(_field, range_value);
+ if(!validate_op.ok()) {
+ return validate_op;
+ }
+
+ f.values.push_back(range_value);
+ f.comparators.push_back(op_comparator.get());
+ }
+ } else {
+ auto validate_op = filter::validate_numerical_filter_value(_field, filter_value);
+ if(!validate_op.ok()) {
+ return validate_op;
+ }
+
+ f.values.push_back(filter_value);
+ f.comparators.push_back(op_comparator.get());
+ }
+ }
+
+ } else {
+ Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);
+ if(!op_comparator.ok()) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());
+ }
+
+ if(op_comparator.get() == RANGE_INCLUSIVE) {
+ // split the value around range operator to extract bounds
+ std::vector<std::string> range_values;
+ StringUtils::split(raw_value, range_values, filter::RANGE_OPERATOR());
+
+ f.field_name = field_name;
+ for(const std::string& range_value: range_values) {
+ auto validate_op = filter::validate_numerical_filter_value(_field, range_value);
+ if(!validate_op.ok()) {
+ return validate_op;
+ }
+
+ f.values.push_back(range_value);
+ f.comparators.push_back(op_comparator.get());
+ }
+ } else {
+ auto validate_op = filter::validate_numerical_filter_value(_field, raw_value);
+ if(!validate_op.ok()) {
+ return validate_op;
+ }
+ f = {field_name, {raw_value}, {op_comparator.get()}};
+ }
+ }
+ } else if(_field.is_bool()) {
+ NUM_COMPARATOR bool_comparator = EQUALS;
+ size_t filter_value_index = 0;
+
+ if(raw_value[0] == '=') {
+ bool_comparator = EQUALS;
+ while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ } else if(raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
+ bool_comparator = NOT_EQUALS;
+ filter_value_index++;
+ while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ }
+
+ if(filter_value_index != 0) {
+ raw_value = raw_value.substr(filter_value_index);
+ }
+
+ if(filter_value_index == raw_value.size()) {
+ return Option<bool>(400, "Error with filter field `" + _field.name +
+ "`: Filter value cannot be empty.");
+ }
+
+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+ f = {field_name, {}, {}};
+
+ for(std::string & filter_value: filter_values) {
+ if(filter_value != "true" && filter_value != "false") {
+ return Option<bool>(400, "Values of filter field `" + _field.name +
+ "`: must be `true` or `false`.");
+ }
+
+ filter_value = (filter_value == "true") ? "1" : "0";
+ f.values.push_back(filter_value);
+ f.comparators.push_back(bool_comparator);
+ }
+ } else {
+ if(raw_value != "true" && raw_value != "false") {
+ return Option<bool>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");
+ }
+
+ std::string bool_value = (raw_value == "true") ? "1" : "0";
+ f = {field_name, {bool_value}, {bool_comparator}};
+ }
+
+ } else if(_field.is_geopoint()) {
+ f = {field_name, {}, {}};
+
+ const std::string& format_err_msg = "Value of filter field `" + _field.name +
+ "`: must be in the `(-44.50, 170.29, 0.75 km)` or "
+ "(56.33, -65.97, 23.82, -127.82) format.";
+
+ NUM_COMPARATOR num_comparator;
+
+ // could be a single value or a list
+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, "),");
+
+ for(std::string& filter_value: filter_values) {
+ filter_value += ")";
+ std::string processed_filter_val;
+ auto parse_op = parse_geopoint_filter_value(filter_value, format_err_msg, processed_filter_val, num_comparator);
+
+ if(!parse_op.ok()) {
+ return parse_op;
+ }
+
+ f.values.push_back(processed_filter_val);
+ f.comparators.push_back(num_comparator);
+ }
+ } else {
+ // single value, e.g. (10.45, 34.56, 2 km)
+ std::string processed_filter_val;
+ auto parse_op = parse_geopoint_filter_value(raw_value, format_err_msg, processed_filter_val, num_comparator);
+
+ if(!parse_op.ok()) {
+ return parse_op;
+ }
+
+ f.values.push_back(processed_filter_val);
+ f.comparators.push_back(num_comparator);
+ }
+
+ } else if(_field.is_string()) {
+ size_t filter_value_index = 0;
+ NUM_COMPARATOR str_comparator = CONTAINS;
+
+ if(raw_value[0] == '=') {
+ if(!_field.facet) {
+ // EQUALS filtering on string is possible only on facet fields
+ return Option<bool>(400, "To perform exact filtering, filter field `" +
+ _field.name + "` must be a facet field.");
+ }
+
+ // string filter should be evaluated in strict "equals" mode
+ str_comparator = EQUALS;
+ while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ } else if(raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
+ if(!_field.facet) {
+ // EXCLUDE filtering on string is possible only on facet fields
+ return Option<bool>(400, "To perform exclude filtering, filter field `" +
+ _field.name + "` must be a facet field.");
+ }
+
+ str_comparator = NOT_EQUALS;
+ filter_value_index++;
+ while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ }
+
+ if(filter_value_index == raw_value.size()) {
+ return Option<bool>(400, "Error with filter field `" + _field.name +
+ "`: Filter value cannot be empty.");
+ }
+
+ if(raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split_to_values(raw_value.substr(filter_value_index+1, raw_value.size() - filter_value_index - 2), filter_values);
+ f = {field_name, filter_values, {str_comparator}};
+ } else {
+ f = {field_name, {raw_value.substr(filter_value_index)}, {str_comparator}};
+ }
+ } else {
+ return Option<bool>(400, "Error with filter field `" + _field.name +
+ "`: Unidentified field data type, see docs for supported data types.");
+ }
+
+ if(f.comparators.size() > 0 && f.comparators.front() == NOT_EQUALS) {
+ exclude_filters.push_back(f);
+ } else {
+ filters.push_back(f);
+ }
+ }
+
+ filters.insert( filters.end(), exclude_filters.begin(), exclude_filters.end() );
+
+ return Option<bool>(true);
+}
diff --git a/src/index.cpp b/src/index.cpp
index c61f9981..9c81aca4 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -24,11 +24,11 @@ spp::sparse_hash_map<uint32_t, int64_t> Index::text_match_sentinel_value;
spp::sparse_hash_map<uint32_t, int64_t> Index::seq_id_sentinel_value;
spp::sparse_hash_map<uint32_t, int64_t> Index::geo_sentinel_value;
-Index::Index(const std::string& name, ThreadPool* thread_pool,
+Index::Index(const std::string& name, const uint32_t collection_id, const Store* store, ThreadPool* thread_pool,
const std::unordered_map<std::string, field> & search_schema,
const std::map<std::string, field>& facet_schema, const std::unordered_map<std::string, field>& sort_schema,
const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators):
- name(name), thread_pool(thread_pool),
+ name(name), collection_id(collection_id), thread_pool(thread_pool),
search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema),
symbols_to_index(symbols_to_index), token_separators(token_separators) {
@@ -1071,13 +1071,11 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
}
}
-uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) const {
+void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,
+ const std::vector<filter>& filters) const {
//auto begin = std::chrono::high_resolution_clock::now();
std::shared_lock lock(mutex);
- uint32_t* filter_ids = nullptr;
- uint32_t filter_ids_length = 0;
-
for(size_t i = 0; i < filters.size(); i++) {
const filter & a_filter = filters[i];
@@ -1422,9 +1420,6 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
LOG(INFO) << "Time taken for filtering: " << timeMillis << "ms";*/
-
- *filter_ids_out = filter_ids;
- return filter_ids_length;
}
void Index::run_search(search_args* search_params) {
@@ -1441,7 +1436,7 @@ void Index::run_search(search_args* search_params) {
search_params->raw_result_kvs, search_params->override_result_kvs,
search_params->typo_tokens_threshold,
search_params->group_limit, search_params->group_by_fields,
- search_params->dynamic_overrides,
+ search_params->filter_overrides,
search_params->default_sorting_field,
search_params->prioritize_exact_match,
search_params->exhaustive_search,
@@ -1521,49 +1516,193 @@ void Index::concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t,
}
}
-void Index::process_dynamic_overrides(const std::vector<const override_t*>& dynamic_overrides,
- std::vector<query_tokens_t>& field_query_tokens,
- token_ordering token_order,
- uint32_t** filter_ids,
- uint32_t& filter_ids_length) const {
-
- // compute dynamic filters if any
- for(auto& override: dynamic_overrides) {
- // will do an AND across different fields
- uint32_t* field_override_ids = nullptr;
- size_t field_override_ids_len = 0;
-
- // check for dynamic filter
- for(auto& field_name: override->dynamic_filters) {
- // check if query matches a filter field
- std::vector<std::string> tokens = field_query_tokens[0].q_include_tokens;
- check_for_overrides(token_order, field_name, override->mutate_query_string,
- field_override_ids, field_override_ids_len, tokens);
-
- for(size_t i=0; i < field_query_tokens.size(); i++) {
- if(tokens.empty()) {
- field_query_tokens[i].q_include_tokens = {"*"};
- } else {
- field_query_tokens[i].q_include_tokens = tokens;
+void Index::static_filter_query_eval(const override_t* override,
+ std::vector<std::string>& tokens,
+ std::vector<filter>& filters) const {
+
+ std::string query = StringUtils::join(tokens, " ");
+
+ if( (override->rule.match == override_t::MATCH_EXACT && override->rule.query == query) ||
+ (override->rule.match == override_t::MATCH_CONTAINS && query.find(override->rule.query) != std::string::npos) ) {
+
+ Option<bool> filter_op = filter::parse_filter_query(override->filter_by, search_schema,
+ store, "", filters);
+ if(!filter_op.ok()) {
+ return ;
+ }
+
+ if(override->mutate_query_string) {
+ query.replace(query.find(override->rule.query), override->rule.query.size(), "");
+ if(StringUtils::trim(query).empty()) {
+ tokens = {"*"};
+ } else {
+ tokens.clear();
+ StringUtils::split(query, tokens, " ");
+ }
+ }
+ }
+}
+
+bool Index::resolve_override(const std::vector<std::string>& rule_parts, const bool exact_rule_match,
+ const std::vector<std::string>& query_tokens,
+ token_ordering token_order, std::set<std::string>& absorbed_tokens,
+ uint32_t*& override_ids, size_t& override_ids_len) const {
+
+ bool resolved_override = false;
+ size_t i = 0, j = 0;
+
+ while(i < rule_parts.size()) {
+ if(rule_parts[i].front() == '{' && rule_parts[i].back() == '}') {
+ // found a field placeholder
+ std::vector<std::string> field_names;
+ std::string rule_part = rule_parts[i];
+ field_names.emplace_back(rule_part.erase(0, 1).erase(rule_part.size() - 1));
+
+ // skip until we find a non-placeholder token
+ i++;
+
+ while(i < rule_parts.size() && (rule_parts[i].front() == '{' && rule_parts[i].back() == '}')) {
+ rule_part = rule_parts[i];
+ field_names.emplace_back(rule_part.erase(0, 1).erase(rule_part.size() - 1));
+ i++;
+ }
+
+ // `i` now points to either end of array or at a non-placeholder rule token
+ // we will skip query tokens until it matches the rule token
+
+ std::vector<std::string> matched_tokens;
+
+ while(j < query_tokens.size() && (i == rule_parts.size() || rule_parts[i] != query_tokens[j])) {
+ matched_tokens.emplace_back(query_tokens[j]);
+ j++;
+ }
+
+ if(i < rule_parts.size() && j < query_tokens.size() && rule_parts[i] != query_tokens[j]) {
+ // if last token does not match, it means that the query does not match the rule
+ resolved_override = false;
+ goto RETURN_EARLY;
+ }
+
+ if(field_names.empty()) {
+ resolved_override = false;
+ goto RETURN_EARLY;
+ }
+
+ resolved_override = true;
+
+ // we try to map `field_names` against `matched_tokens` now
+ for(size_t findex = 0; findex < field_names.size(); findex++) {
+ const auto& field_name = field_names[findex];
+ bool slide_window = (findex == 0); // fields following another field should match exactly
+ resolved_override &= check_for_overrides2(token_order, field_name, slide_window,
+ override_ids, override_ids_len,
+ exact_rule_match, matched_tokens, absorbed_tokens);
+ if(!resolved_override) {
+ goto RETURN_EARLY;
+ }
+ }
+ } else {
+ i++;
+ j++;
+ }
+ }
+
+ RETURN_EARLY:
+
+ if(!resolved_override || (exact_rule_match && query_tokens.size() != absorbed_tokens.size())) {
+ delete [] override_ids;
+ override_ids_len = 0;
+ return false;
+ }
+
+ return resolved_override;
+}
+
+void Index::process_filter_overrides(const std::vector<const override_t*>& filter_overrides,
+ std::vector<query_tokens_t>& field_query_tokens,
+ token_ordering token_order,
+ std::vector<filter>& filters,
+ uint32_t** filter_ids,
+ uint32_t& filter_ids_length) const {
+
+ for(auto& override: filter_overrides) {
+ if(!override->rule.dynamic_query) {
+ // simple static filtering: add to filter_by and rewrite query if needed
+ // we will cover both the original query and the synonym variants
+ size_t orig_filters_size = filters.size();
+
+ static_filter_query_eval(override, field_query_tokens[0].q_include_tokens, filters);
+ for(auto& syn_tokens: field_query_tokens[0].q_synonyms) {
+ static_filter_query_eval(override, syn_tokens, filters);
+ }
+
+ if(orig_filters_size != filters.size()) {
+ // means that we have been able to resolve an override
+ return ;
+ }
+ } else {
+ // need to extract placeholder field names from the search query, filter on them and rewrite query
+ // we will again cover both original query and synonyms
+
+ std::vector<std::string> rule_parts;
+ StringUtils::split(override->rule.query, rule_parts, " ");
+
+ std::vector<std::string>& query_tokens = field_query_tokens[0].q_include_tokens;
+
+ uint32_t* field_override_ids = nullptr;
+ size_t field_override_ids_len = 0;
+ bool exact_rule_match = override->rule.match == override_t::MATCH_EXACT;
+
+ std::set<std::string> absorbed_tokens;
+ bool resolved_override = resolve_override(rule_parts, exact_rule_match, query_tokens, token_order,
+ absorbed_tokens, field_override_ids, field_override_ids_len);
+
+ if(resolved_override && override->mutate_query_string) {
+ std::vector<std::string> new_tokens;
+ for(auto& token: query_tokens) {
+ if(absorbed_tokens.count(token) == 0) {
+ new_tokens.emplace_back(token);
+ }
+ }
+
+ for(size_t j=0; j < field_query_tokens.size(); j++) {
+ if(new_tokens.empty()) {
+ field_query_tokens[j].q_include_tokens = {"*"};
+ } else {
+ field_query_tokens[j].q_include_tokens = new_tokens;
+ }
}
}
+ // now resolve synonyms
+
for(size_t i = 0; i < field_query_tokens[0].q_synonyms.size(); i++) {
uint32_t* synonym_override_ids = nullptr;
size_t synonym_override_ids_len = 0;
+ absorbed_tokens.clear();
- std::vector<std::string> syn_tokens = field_query_tokens[0].q_synonyms[i];
- check_for_overrides(token_order, field_name, override->mutate_query_string,
- synonym_override_ids, synonym_override_ids_len,syn_tokens);
+ std::vector<std::string>& syn_tokens = field_query_tokens[0].q_synonyms[i];
+ resolved_override = resolve_override(rule_parts, exact_rule_match, syn_tokens, token_order, absorbed_tokens,
+ synonym_override_ids, synonym_override_ids_len);
- for(size_t j=0; j < field_query_tokens.size(); j++) {
- if(syn_tokens.empty()) {
- field_query_tokens[j].q_synonyms[i] = {"*"};
- } else {
- field_query_tokens[j].q_synonyms[i] = syn_tokens;
+ if(resolved_override && override->mutate_query_string) {
+ std::vector<std::string> new_tokens;
+ for (auto& token: syn_tokens) {
+ if (absorbed_tokens.count(token) == 0) {
+ new_tokens.emplace_back(token);
+ }
+ }
+
+ for(size_t j=0; j < field_query_tokens.size(); j++) {
+ if(new_tokens.empty()) {
+ field_query_tokens[j].q_synonyms[i] = {"*"};
+ } else {
+ field_query_tokens[j].q_synonyms[i] = new_tokens;
+ }
}
}
+
// result ids of synonyms will be ORed
uint32_t* syn_field_override_ids = nullptr;
@@ -1575,31 +1714,31 @@ void Index::process_dynamic_overrides(const std::vector<const override_t*>& dyna
delete [] field_override_ids;
field_override_ids = syn_field_override_ids;
}
- }
- if(filter_ids_length != 0) {
- uint32_t* filtered_results = nullptr;
- filter_ids_length = ArrayUtils::and_scalar(field_override_ids, field_override_ids_len, *filter_ids,
- filter_ids_length, &filtered_results);
- delete [] *filter_ids;
- delete [] field_override_ids;
- *filter_ids = filtered_results;
- } else {
- *filter_ids = field_override_ids;
- filter_ids_length = field_override_ids_len;
+ if(field_override_ids_len != 0) {
+ if(filter_ids_length != 0) {
+ uint32_t* filtered_results = nullptr;
+ filter_ids_length = ArrayUtils::and_scalar(field_override_ids, field_override_ids_len, *filter_ids,
+ filter_ids_length, &filtered_results);
+ delete [] *filter_ids;
+ delete [] field_override_ids;
+ *filter_ids = filtered_results;
+ } else {
+ *filter_ids = field_override_ids;
+ filter_ids_length = field_override_ids_len;
+ }
+
+ return ;
+ }
}
}
}
-void
-Index::check_for_overrides(const token_ordering& token_order, const string& field_name, bool mutate_query_string,
- uint32_t*& field_override_ids,
- size_t& field_override_ids_len, std::vector<std::string>& actual_tokens) const {
-
- std::vector<std::string> local_tokens = actual_tokens;
- std::vector<std::string>& tokens = mutate_query_string ? actual_tokens : local_tokens;
+bool Index::check_for_overrides2(const token_ordering& token_order, const string& field_name, const bool slide_window,
+ uint32_t*& field_override_ids, size_t& field_override_ids_len,
+ bool exact_rule_match, std::vector<std::string>& tokens,
+ std::set<std::string>& absorbed_tokens) const {
- DO_WINDOWING:
for(size_t window_len = tokens.size(); window_len > 0; window_len--) {
for(size_t start_index = 0; start_index+window_len-1 < tokens.size(); start_index++) {
std::vector<token_t> window_tokens;
@@ -1640,24 +1779,33 @@ Index::check_for_overrides(const token_ordering& token_order, const string& fiel
field_override_ids_len = result_ids_len;
}
- // remove window_tokens from `tokens` and repeat search
+ // remove window_tokens from `tokens`
std::vector<std::string> new_tokens;
- for(const auto& token: tokens) {
+ for(size_t new_i = start_index; new_i < tokens.size(); new_i++) {
+ const auto& token = tokens[new_i];
if(window_tokens_set.count(token) == 0) {
new_tokens.emplace_back(token);
+ } else {
+ absorbed_tokens.insert(token);
}
}
tokens = new_tokens;
- goto DO_WINDOWING;
+ return true;
+ }
+
+ if(!slide_window) {
+ break;
}
}
}
+
+ return false;
}
void Index::search(std::vector<query_tokens_t>& field_query_tokens,
const std::vector<search_field_t>& search_fields,
- const std::vector<filter>& filters,
+ std::vector<filter>& filters,
std::vector<facet>& facets, facet_query_t& facet_query,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
const std::vector<uint32_t> & excluded_ids,
@@ -1674,7 +1822,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,
const size_t typo_tokens_threshold,
const size_t group_limit,
const std::vector<std::string>& group_by_fields,
- const std::vector<const override_t*>& dynamic_overrides,
+ const std::vector<const override_t*>& filter_overrides,
const std::string& default_sorting_field,
bool prioritize_exact_match,
const bool exhaustive_search,
@@ -1685,7 +1833,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,
// process the filters
uint32_t* filter_ids = nullptr;
- uint32_t filter_ids_length = do_filtering(&filter_ids, filters);
+ uint32_t filter_ids_length = 0;
std::shared_lock lock(mutex);
@@ -1705,7 +1853,14 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,
std::vector<uint32_t> curated_ids_sorted(curated_ids.begin(), curated_ids.end());
std::sort(curated_ids_sorted.begin(), curated_ids_sorted.end());
- process_dynamic_overrides(dynamic_overrides, field_query_tokens, token_order, &filter_ids, filter_ids_length);
+ process_filter_overrides(filter_overrides, field_query_tokens, token_order, filters,
+ &filter_ids, filter_ids_length);
+
+ lock.unlock();
+
+ do_filtering(filter_ids, filter_ids_length, filters);
+
+ lock.lock();
// Order of `fields` are used to sort results
//auto begin = std::chrono::high_resolution_clock::now();
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index eff9f8a8..bd8c91fd 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -262,7 +262,7 @@ TEST_F(CollectionOverrideTest, OverrideJSONValidation) {
parse_op = override_t::parse(include_json2, "", override2);
ASSERT_FALSE(parse_op.ok());
- ASSERT_STREQ("Must contain one of:`includes`, `excludes`, `dynamic_filters`.", parse_op.error().c_str());
+ ASSERT_STREQ("Must contain one of:`includes`, `excludes`, `filter_by`.", parse_op.error().c_str());
include_json2["includes"] = nlohmann::json::array();
include_json2["includes"][0] = 100;
@@ -817,36 +817,60 @@ TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {
doc2["brand"] = "Adidas";
doc2["points"] = 5;
+ nlohmann::json doc3;
+ doc3["id"] = "2";
+ doc3["name"] = "Running Shoes";
+ doc3["category"] = "sports";
+ doc3["brand"] = "Nike";
+ doc3["points"] = 5;
+
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());
std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
auto results = coll1->search("shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10).get();
- ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("2", results["hits"][2]["document"]["id"].get<std::string>());
// with override, results will be different
nlohmann::json override_json = {
- {"id", "dynamic-filters"},
+ {"id", "dynamic-cat-filter"},
{
"rule", {
- {"query", ".*"},
- {"match", override_t::MATCH_EXACT}
+ {"query", "{category}"},
+ {"match", override_t::MATCH_EXACT}
}
},
{"mutate_query_string", true},
- {"dynamic_filters", "category,brand"}
+ {"filter_by", "category: {category}"}
};
override_t override;
- auto op = override_t::parse(override_json, "dynamic-filters", override);
+ auto op = override_t::parse(override_json, "dynamic-cat-filter", override);
ASSERT_TRUE(op.ok());
+ coll1->add_override(override);
+ override_json = {
+ {"id", "dynamic-brand-cat-filter"},
+ {
+ "rule", {
+ {"query", "{brand} {category}"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ },
+ {"mutate_query_string", true},
+ {"filter_by", "category: {category} && brand: {brand}"}
+ };
+
+ op = override_t::parse(override_json, "dynamic-brand-cat-filter", override);
+ ASSERT_TRUE(op.ok());
coll1->add_override(override);
results = coll1->search("shoes", {"name", "category", "brand"}, "",
@@ -856,6 +880,12 @@ TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+ // should not apply filter for non-exact case
+ results = coll1->search("running shoes", {"name", "category", "brand"}, "",
+ {}, sort_fields, {2, 2, 2}, 10).get();
+
+ ASSERT_EQ(3, results["hits"].size());
+
results = coll1->search("adidas shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10).get();
@@ -868,35 +898,185 @@ TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {
{"id", "dynamic-filters-bad1"},
{
"rule", {
- {"query", ".*"},
- {"match", override_t::MATCH_EXACT}
+ {"query", "{brand}"},
+ {"match", override_t::MATCH_EXACT}
}
},
{"mutate_query_string", true},
- {"dynamic_filters", ""}
+ {"filter_by", ""}
};
override_t override_bad1;
op = override_t::parse(override_json_bad1, "dynamic-filters-bad1", override_bad1);
ASSERT_FALSE(op.ok());
- ASSERT_EQ("The `dynamic_filters` must be a comma separated string of filter field names.", op.error());
+ ASSERT_EQ("The `filter_by` must be a non-empty string.", op.error());
nlohmann::json override_json_bad2 = {
{"id", "dynamic-filters-bad2"},
{
"rule", {
- {"query", ".*"},
+ {"query", "{brand}"},
{"match", override_t::MATCH_EXACT}
}
},
{"mutate_query_string", true},
- {"dynamic_filters", {"foo", "bar"}}
+ {"filter_by", {"foo", "bar"}}
};
override_t override_bad2;
op = override_t::parse(override_json_bad2, "dynamic-filters-bad2", override_bad2);
ASSERT_FALSE(op.ok());
- ASSERT_EQ("The `dynamic_filters` must be a comma separated string of filter field names.", op.error());
+ ASSERT_EQ("The `filter_by` must be a string.", op.error());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionOverrideTest, DynamicFilteringMultiplePlaceholders) {
+ Collection* coll1;
+
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("category", field_types::STRING, true),
+ field("brand", field_types::STRING, true),
+ field("color", field_types::STRING, true),
+ field("points", field_types::INT32, false)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Retro Shoes";
+ doc1["category"] = "shoes";
+ doc1["color"] = "yellow";
+ doc1["brand"] = "Nike Air Jordan";
+ doc1["points"] = 3;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "Baseball";
+ doc2["category"] = "shoes";
+ doc2["color"] = "white";
+ doc2["brand"] = "Adidas";
+ doc2["points"] = 5;
+
+ nlohmann::json doc3;
+ doc3["id"] = "2";
+ doc3["name"] = "Running Shoes";
+ doc3["category"] = "sports";
+ doc3["color"] = "grey";
+ doc3["brand"] = "Nike";
+ doc3["points"] = 5;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());
+
+ std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC"), sort_by("points", "DESC")};
+
+ nlohmann::json override_json = {
+ {"id", "dynamic-cat-filter"},
+ {
+ "rule", {
+ {"query", "{brand} {color} shoes"},
+ {"match", override_t::MATCH_CONTAINS}
+ }
+ },
+ {"mutate_query_string", true},
+ {"filter_by", "brand: {brand} && color: {color}"}
+ };
+
+ override_t override;
+ auto op = override_t::parse(override_json, "dynamic-cat-filter", override);
+ ASSERT_TRUE(op.ok());
+ coll1->add_override(override);
+
+ // not an exact match of rule (because of "light") so all results will be fetched, not just Air Jordan brand
+ auto results = coll1->search("Nike Air Jordan light yellow shoes", {"name", "category", "brand"}, "",
+ {}, sort_fields, {2, 2, 2}, 10).get();
+
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());
+
+ // query with tokens at the start that preceding the placeholders in the rule
+ results = coll1->search("New Nike Air Jordan yellow shoes", {"name", "category", "brand"}, "",
+ {}, sort_fields, {2, 2, 2}, 10).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionOverrideTest, DynamicFilteringTokensBetweenPlaceholders) {
+ Collection* coll1;
+
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("category", field_types::STRING, true),
+ field("brand", field_types::STRING, true),
+ field("color", field_types::STRING, true),
+ field("points", field_types::INT32, false)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Retro Shoes";
+ doc1["category"] = "shoes";
+ doc1["color"] = "yellow";
+ doc1["brand"] = "Nike Air Jordan";
+ doc1["points"] = 3;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "Baseball";
+ doc2["category"] = "shoes";
+ doc2["color"] = "white";
+ doc2["brand"] = "Adidas";
+ doc2["points"] = 5;
+
+ nlohmann::json doc3;
+ doc3["id"] = "2";
+ doc3["name"] = "Running Shoes";
+ doc3["category"] = "sports";
+ doc3["color"] = "grey";
+ doc3["brand"] = "Nike";
+ doc3["points"] = 5;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());
+
+ std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC"), sort_by("points", "DESC")};
+
+ nlohmann::json override_json = {
+ {"id", "dynamic-cat-filter"},
+ {
+ "rule", {
+ {"query", "{brand} shoes {color}"},
+ {"match", override_t::MATCH_CONTAINS}
+ }
+ },
+ {"mutate_query_string", true},
+ {"filter_by", "brand: {brand} && color: {color}"}
+ };
+
+ override_t override;
+ auto op = override_t::parse(override_json, "dynamic-cat-filter", override);
+ ASSERT_TRUE(op.ok());
+ coll1->add_override(override);
+
+ auto results = coll1->search("Nike Air Jordan shoes yellow", {"name", "category", "brand"}, "",
+ {}, sort_fields, {2, 2, 2}, 10).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
@@ -952,12 +1132,12 @@ TEST_F(CollectionOverrideTest, DynamicFilteringWithSynonyms) {
{"id", "dynamic-filters"},
{
"rule", {
- {"query", ".*"},
+ {"query", "{category}"},
{"match", override_t::MATCH_EXACT}
}
},
{"mutate_query_string", true},
- {"dynamic_filters", "category,brand"}
+ {"filter_by", "category: {category}"}
};
override_t override;
@@ -1025,7 +1205,7 @@ TEST_F(CollectionOverrideTest, StaticFiltering) {
}
},
{"mutate_query_string", true},
- {"dynamic_filters", "price:> 100"}
+ {"filter_by", "price:> 100"}
};
override_t override_contains;
@@ -1043,7 +1223,7 @@ TEST_F(CollectionOverrideTest, StaticFiltering) {
}
},
{"mutate_query_string", true},
- {"dynamic_filters", "price:< 100"}
+ {"filter_by", "price:< 100"}
};
override_t override_exact;
@@ -1127,7 +1307,7 @@ TEST_F(CollectionOverrideTest, FilteringWithAndWithoutQueryStringMutation) {
}
},
{"mutate_query_string", false},
- {"dynamic_filters", "price:> 200"}
+ {"filter_by", "price:> 200"}
};
override_t override_contains;
@@ -1155,7 +1335,7 @@ TEST_F(CollectionOverrideTest, FilteringWithAndWithoutQueryStringMutation) {
}
},
{"mutate_query_string", true},
- {"dynamic_filters", "price:> 200"}
+ {"filter_by", "price:> 200"}
};
op = override_t::parse(override_json_contains, "static-filters", override_contains);
diff --git a/test/index_test.cpp b/test/index_test.cpp
index 8ca0ef07..b06e8ae8 100644
--- a/test/index_test.cpp
+++ b/test/index_test.cpp
@@ -12,7 +12,7 @@ TEST(IndexTest, ScrubReindexDoc) {
ThreadPool pool(4);
- Index index("index", &pool, search_schema, {}, {}, {}, {});
+ Index index("index", 1, nullptr, &pool, search_schema, {}, {}, {}, {});
nlohmann::json old_doc;
old_doc["id"] = "1";
old_doc["title"] = "One more thing.";