Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Enable filtering via overrides.

Kishore Nallanc6fe1369b957
diff --git a/include/collection.h b/include/collection.h
index f6e5a244..b1697f2b 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -19,151 +19,6 @@
#include <option.h>
#include "tokenizer.h"
-
-struct override_t {
- static const std::string MATCH_EXACT;
- static const std::string MATCH_CONTAINS;
-
- struct rule_t {
- std::string query;
- std::string match;
- };
-
- struct add_hit_t {
- std::string doc_id;
- uint32_t position;
- };
-
- struct drop_hit_t {
- std::string doc_id;
- };
-
- std::string id;
- rule_t rule;
- std::vector<add_hit_t> add_hits;
- std::vector<drop_hit_t> drop_hits;
-
- override_t() {}
-
- static Option<bool> parse(const nlohmann::json& override_json, const std::string& id, override_t& override) {
- if(!override_json.is_object()) {
- return Option<bool>(400, "Bad JSON.");
- }
-
- if(override_json.count("rule") == 0 || !override_json["rule"].is_object()) {
- return Option<bool>(400, "Missing `rule` definition.");
- }
-
- if(override_json["rule"].count("query") == 0 || override_json["rule"].count("match") == 0) {
- return Option<bool>(400, "The `rule` definition must contain a `query` or `match`.");
- }
-
- if(override_json.count("includes") == 0 && override_json.count("excludes") == 0) {
- return Option<bool>(400, "Must contain either `includes` or `excludes`.");
- }
-
- if(override_json.count("includes") != 0) {
- if(!override_json["includes"].is_array()) {
- return Option<bool>(400, "The `includes` value must be an array.");
- }
-
- for(const auto & include_obj: override_json["includes"]) {
- if(!include_obj.is_object()) {
- return Option<bool>(400, "The `includes` value must be an array of objects.");
- }
-
- if(include_obj.count("id") == 0 || include_obj.count("position") == 0) {
- return Option<bool>(400, "Inclusion definition must define both `id` and `position` keys.");
- }
-
- if(!include_obj["id"].is_string()) {
- return Option<bool>(400, "Inclusion `id` must be a string.");
- }
-
- if(!include_obj["position"].is_number_integer()) {
- return Option<bool>(400, "Inclusion `position` must be an integer.");
- }
- }
- }
-
- if(override_json.count("excludes") != 0) {
- if(!override_json["excludes"].is_array()) {
- return Option<bool>(400, "The `excludes` value must be an array.");
- }
-
- for(const auto & exclude_obj: override_json["excludes"]) {
- if(!exclude_obj.is_object()) {
- return Option<bool>(400, "The `excludes` value must be an array of objects.");
- }
-
- if(exclude_obj.count("id") == 0) {
- return Option<bool>(400, "Exclusion definition must define an `id`.");
- }
-
- if(!exclude_obj["id"].is_string()) {
- return Option<bool>(400, "Exclusion `id` must be a string.");
- }
- }
-
- }
-
- if(!id.empty()) {
- override.id = id;
- } else if(override_json.count("id") != 0) {
- override.id = override_json["id"].get<std::string>();
- } else {
- return Option<bool>(400, "Override `id` not provided.");
- }
-
- override.rule.query = override_json["rule"]["query"].get<std::string>();
- override.rule.match = override_json["rule"]["match"].get<std::string>();
-
- if (override_json.count("includes") != 0) {
- for(const auto & include: override_json["includes"]) {
- add_hit_t add_hit;
- add_hit.doc_id = include["id"].get<std::string>();
- add_hit.position = include["position"].get<uint32_t>();
- override.add_hits.push_back(add_hit);
- }
- }
-
- if (override_json.count("excludes") != 0) {
- for(const auto & exclude: override_json["excludes"]) {
- drop_hit_t drop_hit;
- drop_hit.doc_id = exclude["id"].get<std::string>();
- override.drop_hits.push_back(drop_hit);
- }
- }
-
- return Option<bool>(true);
- }
-
- nlohmann::json to_json() const {
- nlohmann::json override;
- override["id"] = id;
- override["rule"]["query"] = rule.query;
- override["rule"]["match"] = rule.match;
-
- override["includes"] = nlohmann::json::array();
-
- for(const auto & add_hit: add_hits) {
- nlohmann::json include;
- include["id"] = add_hit.doc_id;
- include["position"] = add_hit.position;
- override["includes"].push_back(include);
- }
-
- override["excludes"] = nlohmann::json::array();
- for(const auto & drop_hit: drop_hits) {
- nlohmann::json exclude;
- exclude["id"] = drop_hit.doc_id;
- override["excludes"].push_back(exclude);
- }
-
- return override;
- }
-};
-
struct doc_seq_id_t {
uint32_t seq_id;
bool is_new;
@@ -348,12 +203,12 @@ private:
void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);
- void curate_results(std::string query,
+ void curate_results(string& actual_query, bool enable_overrides, bool already_segmented,
const std::map<size_t, std::vector<std::string>>& pinned_hits,
const std::vector<std::string>& hidden_hits,
std::map<size_t, std::vector<uint32_t>>& include_ids,
- std::vector<uint32_t> & excluded_ids,
- bool enable_overrides) const;
+ std::vector<uint32_t>& excluded_ids, std::vector<const override_t*>& dynamic_overrides,
+ std::vector<filter>& filters) const;
Option<bool> check_and_update_schema(nlohmann::json& document, const DIRTY_VALUES& dirty_values);
diff --git a/include/index.h b/include/index.h
index 97c29dec..3c0042e4 100644
--- a/include/index.h
+++ b/include/index.h
@@ -47,6 +47,181 @@ struct query_tokens_t {
std::vector<std::vector<std::string>> q_synonyms;
};
+struct override_t {
+ static const std::string MATCH_EXACT;
+ static const std::string MATCH_CONTAINS;
+
+ struct rule_t {
+ std::string query;
+ std::string match;
+ };
+
+ struct add_hit_t {
+ std::string doc_id;
+ uint32_t position;
+ };
+
+ struct drop_hit_t {
+ std::string doc_id;
+ };
+
+ std::string id;
+
+ rule_t rule;
+ std::vector<add_hit_t> add_hits;
+ std::vector<drop_hit_t> drop_hits;
+
+ std::vector<std::string> dynamic_filters;
+ bool mutate_query_string = false;
+
+ override_t() {}
+
+ static Option<bool> parse(const nlohmann::json& override_json, const std::string& id, override_t& override) {
+ if(!override_json.is_object()) {
+ return Option<bool>(400, "Bad JSON.");
+ }
+
+ if(override_json.count("rule") == 0 || !override_json["rule"].is_object()) {
+ return Option<bool>(400, "Missing `rule` definition.");
+ }
+
+ if(override_json["rule"].count("query") == 0 || override_json["rule"].count("match") == 0) {
+ return Option<bool>(400, "The `rule` definition must contain a `query` and `match`.");
+ }
+
+ if(override_json.count("includes") == 0 && override_json.count("excludes") == 0 &&
+ override_json.count("dynamic_filters") == 0) {
+ return Option<bool>(400, "Must contain one of:`includes`, `excludes`, `dynamic_filters`.");
+ }
+
+ if(override_json.count("includes") != 0) {
+ if(!override_json["includes"].is_array()) {
+ return Option<bool>(400, "The `includes` value must be an array.");
+ }
+
+ for(const auto & include_obj: override_json["includes"]) {
+ if(!include_obj.is_object()) {
+ return Option<bool>(400, "The `includes` value must be an array of objects.");
+ }
+
+ if(include_obj.count("id") == 0 || include_obj.count("position") == 0) {
+ return Option<bool>(400, "Inclusion definition must define both `id` and `position` keys.");
+ }
+
+ if(!include_obj["id"].is_string()) {
+ return Option<bool>(400, "Inclusion `id` must be a string.");
+ }
+
+ if(!include_obj["position"].is_number_integer()) {
+ return Option<bool>(400, "Inclusion `position` must be an integer.");
+ }
+ }
+ }
+
+ if(override_json.count("excludes") != 0) {
+ if(!override_json["excludes"].is_array()) {
+ return Option<bool>(400, "The `excludes` value must be an array.");
+ }
+
+ for(const auto & exclude_obj: override_json["excludes"]) {
+ if(!exclude_obj.is_object()) {
+ return Option<bool>(400, "The `excludes` value must be an array of objects.");
+ }
+
+ if(exclude_obj.count("id") == 0) {
+ return Option<bool>(400, "Exclusion definition must define an `id`.");
+ }
+
+ if(!exclude_obj["id"].is_string()) {
+ return Option<bool>(400, "Exclusion `id` must be a string.");
+ }
+ }
+
+ }
+
+ if(override_json.count("dynamic_filters") != 0) {
+ if(!override_json["dynamic_filters"].is_string()) {
+ return Option<bool>(400, "The `dynamic_filters` must be "
+ "a comma separated string of filter field names.");
+ }
+
+ if(override_json["dynamic_filters"].get<std::string>().empty()) {
+ return Option<bool>(400, "The `dynamic_filters` must be "
+ "a comma separated string of filter field names.");
+ }
+ }
+
+ if(override_json.count("mutate_query_string") != 0) {
+ if (!override_json["mutate_query_string"].is_boolean()) {
+ return Option<bool>(400, "The `mutate_query_string` must be a boolean.");
+ }
+
+ override.mutate_query_string = override_json["mutate_query_string"].get<bool>();
+ }
+
+ if(!id.empty()) {
+ override.id = id;
+ } else if(override_json.count("id") != 0) {
+ override.id = override_json["id"].get<std::string>();
+ } else {
+ return Option<bool>(400, "Override `id` not provided.");
+ }
+
+ override.rule.query = override_json["rule"]["query"].get<std::string>();
+ override.rule.match = override_json["rule"]["match"].get<std::string>();
+
+ if (override_json.count("includes") != 0) {
+ for(const auto & include: override_json["includes"]) {
+ add_hit_t add_hit;
+ add_hit.doc_id = include["id"].get<std::string>();
+ add_hit.position = include["position"].get<uint32_t>();
+ override.add_hits.push_back(add_hit);
+ }
+ }
+
+ if (override_json.count("excludes") != 0) {
+ for(const auto & exclude: override_json["excludes"]) {
+ drop_hit_t drop_hit;
+ drop_hit.doc_id = exclude["id"].get<std::string>();
+ override.drop_hits.push_back(drop_hit);
+ }
+ }
+
+ if (override_json.count("dynamic_filters") != 0) {
+ std::vector<std::string> filter_fields;
+ StringUtils::split(override_json["dynamic_filters"].get<std::string>(), filter_fields, ",");
+ override.dynamic_filters = filter_fields;
+ }
+
+ return Option<bool>(true);
+ }
+
+ nlohmann::json to_json() const {
+ nlohmann::json override;
+ override["id"] = id;
+ override["rule"]["query"] = rule.query;
+ override["rule"]["match"] = rule.match;
+
+ override["includes"] = nlohmann::json::array();
+
+ for(const auto & add_hit: add_hits) {
+ nlohmann::json include;
+ include["id"] = add_hit.doc_id;
+ include["position"] = add_hit.position;
+ override["includes"].push_back(include);
+ }
+
+ override["excludes"] = nlohmann::json::array();
+ for(const auto & drop_hit: drop_hits) {
+ nlohmann::json exclude;
+ exclude["id"] = drop_hit.doc_id;
+ override["excludes"].push_back(exclude);
+ }
+
+ return override;
+ }
+};
+
struct search_args {
std::vector<query_tokens_t> field_query_tokens;
std::vector<search_field_t> search_fields;
@@ -71,6 +246,7 @@ struct search_args {
size_t all_result_ids_len;
bool exhaustive_search;
size_t concurrency;
+ const std::vector<const override_t*>& dynamic_overrides;
spp::sparse_hash_set<uint64_t> groups_processed;
std::vector<std::vector<art_leaf*>> searched_queries;
@@ -91,7 +267,8 @@ struct search_args {
const std::string& default_sorting_field,
bool prioritize_exact_match,
bool exhaustive_search,
- size_t concurrency):
+ size_t concurrency,
+ const std::vector<const override_t*>& dynamic_overrides):
field_query_tokens(field_query_tokens),
search_fields(search_fields), filters(filters), facets(facets),
included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),
@@ -100,7 +277,8 @@ struct search_args {
drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),
group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),
prioritize_exact_match(prioritize_exact_match), all_result_ids_len(0),
- exhaustive_search(exhaustive_search), concurrency(concurrency) {
+ exhaustive_search(exhaustive_search), concurrency(concurrency),
+ dynamic_overrides(dynamic_overrides) {
const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatory
topster = new Topster(topster_size, group_limit);
@@ -218,6 +396,12 @@ private:
size_t group_limit, const std::vector<std::string>& group_by_fields,
const uint32_t* result_ids, size_t results_size) const;
+ void process_dynamic_overrides(const std::vector<const override_t*>& dynamic_overrides,
+ std::vector<query_tokens_t>& field_query_tokens,
+ const token_ordering token_order,
+ uint32_t** filter_ids,
+ uint32_t& filter_ids_length) const;
+
void search_field(const uint8_t & field_id,
std::vector<token_t>& query_tokens,
std::vector<token_t>& search_tokens,
@@ -374,7 +558,7 @@ public:
void run_search(search_args* search_params);
- void search(const std::vector<query_tokens_t>& field_query_tokens,
+ void search(std::vector<query_tokens_t>& field_query_tokens,
const std::vector<search_field_t> & search_fields,
const std::vector<filter> & filters, std::vector<facet> & facets,
facet_query_t & facet_query,
@@ -392,6 +576,7 @@ public:
const size_t typo_tokens_threshold,
const size_t group_limit,
const std::vector<std::string>& group_by_fields,
+ const std::vector<const override_t*>& dynamic_overrides,
const std::string& default_sorting_field,
bool prioritize_exact_match,
bool exhaustive_search,
@@ -434,5 +619,23 @@ public:
const std::string& fallback_field_type,
const DIRTY_VALUES& dirty_values);
+ void check_for_overrides(const token_ordering& token_order, const string& field_name, bool mutate_query_string,
+ uint32_t*& field_override_ids,
+ size_t& field_override_ids_len, std::vector<std::string>& tokens) const;
+
+ void search_wildcard(const std::vector<std::string>& qtokens, const std::vector<filter>& filters,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ const std::vector<sort_by>& sort_fields_std, Topster* topster, Topster* curated_topster,
+ spp::sparse_hash_set<uint64_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
+ const std::set<uint32_t>& curated_ids, const std::vector<uint32_t>& curated_ids_sorted,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size, const uint8_t field_id,
+ const string& field, uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const uint32_t* filter_ids, uint32_t filter_ids_length) const;
+
+ void curate_filtered_ids(const std::vector<filter>& filters, const std::set<uint32_t>& curated_ids,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size, uint32_t*& filter_ids,
+ uint32_t& filter_ids_length, const std::vector<uint32_t>& curated_ids_sorted) const;
};
diff --git a/src/collection.cpp b/src/collection.cpp
index 1e8268cf..56bb731c 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -373,13 +373,13 @@ void Collection::prune_document(nlohmann::json &document, const spp::sparse_hash
}
}
-void Collection::curate_results(std::string query,
+void Collection::curate_results(string& actual_query, bool enable_overrides, bool already_segmented,
const std::map<size_t, std::vector<std::string>>& pinned_hits,
const std::vector<std::string>& hidden_hits,
std::map<size_t, std::vector<uint32_t>>& include_ids,
- std::vector<uint32_t> & excluded_ids,
- bool enable_overrides) const {
- StringUtils::tolowercase(query);
+ std::vector<uint32_t>& excluded_ids, std::vector<const override_t*>& dynamic_overrides,
+ std::vector<filter>& filters) const {
+
std::set<uint32_t> excluded_set;
// If pinned or hidden hits are provided, they take precedence over overrides
@@ -395,9 +395,19 @@ void Collection::curate_results(std::string query,
}
}
- if(enable_overrides) {
- for(const auto & override_kv: overrides) {
- const auto & override = override_kv.second;
+ std::string query = actual_query;
+
+ if(enable_overrides && !overrides.empty()) {
+ StringUtils::tolowercase(query);
+
+ for(const auto& override_kv: overrides) {
+ const auto& override = override_kv.second;
+
+ // static overrides are applied first as they precedence over dynamic overrides
+ if(!override.dynamic_filters.empty() && override.rule.query == ".*") {
+ dynamic_overrides.push_back(&override);
+ continue;
+ }
if( (override.rule.match == override_t::MATCH_EXACT && override.rule.query == query) ||
(override.rule.match == override_t::MATCH_CONTAINS && query.find(override.rule.query) != std::string::npos) ) {
@@ -422,6 +432,21 @@ void Collection::curate_results(std::string query,
include_ids[hit.position].push_back(seq_id);
}
}
+
+ for(const auto& filter_str: override.dynamic_filters) {
+ auto op = parse_filter_query(filter_str, filters);
+ if(op.ok() && override.mutate_query_string) {
+ actual_query.replace(
+ actual_query.find(override.rule.query),
+ override.rule.query.size(),
+ ""
+ );
+
+ if(StringUtils::trim(actual_query).empty()) {
+ actual_query = "*";
+ }
+ }
+ }
}
}
}
@@ -444,7 +469,7 @@ void Collection::curate_results(std::string query,
}
}
-Option<nlohmann::json> Collection::search(const std::string & query, const std::vector<std::string>& search_fields,
+Option<nlohmann::json> Collection::search(const std::string & raw_query, const std::vector<std::string>& search_fields,
const std::string & simple_filter_query, const std::vector<std::string>& facet_fields,
const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,
const size_t per_page, const size_t page,
@@ -474,7 +499,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
std::shared_lock lock(mutex);
- if(query != "*" && search_fields.empty()) {
+ if(raw_query != "*" && search_fields.empty()) {
return Option<nlohmann::json>(400, "No search fields specified for the query.");
}
@@ -502,57 +527,6 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
}
- std::vector<uint32_t> excluded_ids;
- std::map<size_t, std::vector<uint32_t>> include_ids; // position => list of IDs
- std::map<size_t, std::vector<std::string>> pinned_hits;
-
- Option<bool> pinned_hits_op = parse_pinned_hits(pinned_hits_str, pinned_hits);
-
- if(!pinned_hits_op.ok()) {
- return Option<nlohmann::json>(400, pinned_hits_op.error());
- }
-
- std::vector<std::string> hidden_hits;
- StringUtils::split(hidden_hits_str, hidden_hits, ",");
-
- curate_results(query, pinned_hits, hidden_hits, include_ids, excluded_ids, enable_overrides);
-
- /*for(auto& kv: include_ids) {
- LOG(INFO) << "key: " << kv.first;
- for(auto val: kv.second) {
- LOG(INFO) << val;
- }
- }
-
- LOG(INFO) << "Excludes:";
-
- for(auto id: excluded_ids) {
- LOG(INFO) << id;
- }
-
- LOG(INFO) << "include_ids size: " << include_ids.size();
- for(auto& group: include_ids) {
- for(uint32_t& seq_id: group.second) {
- LOG(INFO) << "seq_id: " << seq_id;
- }
-
- LOG(INFO) << "----";
- }
- */
-
- std::map<size_t, std::map<size_t, uint32_t>> included_ids;
-
- for(const auto& pos_ids: include_ids) {
- size_t outer_pos = pos_ids.first;
- size_t ids_per_pos = std::max(size_t(1), group_limit);
-
- for(size_t inner_pos = 0; inner_pos < std::min(ids_per_pos, pos_ids.second.size()); inner_pos++) {
- auto seq_id = pos_ids.second[inner_pos];
- included_ids[outer_pos][inner_pos] = seq_id;
- //LOG(INFO) << "Adding seq_id " << seq_id << " to index_id " << index_id;
- }
- }
-
// process weights for search fields
std::vector<search_field_t> weighted_search_fields;
size_t max_weight = 20;
@@ -833,7 +807,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
size_t max_hits = 250;
// ensure that `max_hits` never exceeds number of documents in collection
- if(search_fields.size() <= 1 || query == "*") {
+ if(search_fields.size() <= 1 || raw_query == "*") {
max_hits = std::min(std::max((page * per_page), max_hits), get_num_documents());
} else {
max_hits = std::min(std::max((page * per_page), max_hits), get_num_documents());
@@ -853,6 +827,60 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
size_t total_found = 0;
spp::sparse_hash_set<uint64_t> groups_processed; // used to calculate total_found for grouped query
+ std::vector<uint32_t> excluded_ids;
+ std::map<size_t, std::vector<uint32_t>> include_ids; // position => list of IDs
+ std::map<size_t, std::vector<std::string>> pinned_hits;
+
+ Option<bool> pinned_hits_op = parse_pinned_hits(pinned_hits_str, pinned_hits);
+
+ if(!pinned_hits_op.ok()) {
+ return Option<nlohmann::json>(400, pinned_hits_op.error());
+ }
+
+ std::vector<std::string> hidden_hits;
+ StringUtils::split(hidden_hits_str, hidden_hits, ",");
+
+ std::vector<const override_t*> dynamic_overrides;
+ std::string query = raw_query;
+ curate_results(query, enable_overrides, pre_segmented_query, pinned_hits, hidden_hits,
+ include_ids, excluded_ids, dynamic_overrides, filters);
+
+ /*for(auto& kv: include_ids) {
+ LOG(INFO) << "key: " << kv.first;
+ for(auto val: kv.second) {
+ LOG(INFO) << val;
+ }
+ }
+
+ LOG(INFO) << "Excludes:";
+
+ for(auto id: excluded_ids) {
+ LOG(INFO) << id;
+ }
+
+ LOG(INFO) << "include_ids size: " << include_ids.size();
+ for(auto& group: include_ids) {
+ for(uint32_t& seq_id: group.second) {
+ LOG(INFO) << "seq_id: " << seq_id;
+ }
+
+ LOG(INFO) << "----";
+ }
+ */
+
+ std::map<size_t, std::map<size_t, uint32_t>> included_ids;
+
+ for(const auto& pos_ids: include_ids) {
+ size_t outer_pos = pos_ids.first;
+ size_t ids_per_pos = std::max(size_t(1), group_limit);
+
+ for(size_t inner_pos = 0; inner_pos < std::min(ids_per_pos, pos_ids.second.size()); inner_pos++) {
+ auto seq_id = pos_ids.second[inner_pos];
+ included_ids[outer_pos][inner_pos] = seq_id;
+ //LOG(INFO) << "Adding seq_id " << seq_id << " to index_id " << index_id;
+ }
+ }
+
//LOG(INFO) << "Num indices used for querying: " << indices.size();
std::vector<query_tokens_t> field_query_tokens;
@@ -888,8 +916,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
per_page, page, token_order, prefixes,
drop_tokens_threshold, typo_tokens_threshold,
group_by_fields, group_limit, default_sorting_field, prioritize_exact_match,
- exhaustive_search,
- 4);
+ exhaustive_search, 4, dynamic_overrides);
index->run_search(search_params);
@@ -2337,6 +2364,10 @@ Option<bool> Collection::parse_pinned_hits(const std::string& pinned_hits_str,
void Collection::synonym_reduction(const std::vector<std::string>& tokens,
std::vector<std::vector<std::string>>& results) const {
+ if(synonym_definitions.empty()) {
+ return;
+ }
+
std::set<uint64_t> processed_syn_hashes;
synonym_reduction_internal(tokens, tokens.size(), 0, processed_syn_hashes, results);
}
diff --git a/src/index.cpp b/src/index.cpp
index 96d32488..7148047f 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1004,35 +1004,40 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
}
std::vector<uint32_t> result_id_vecs[concurrency];
-
Topster* topsters[concurrency];
- for(size_t i = 0; i < concurrency; i++) {
- topsters[i] = new Topster(topster->MAX_SIZE, topster->distinct);
- }
-
std::vector<spp::sparse_hash_set<uint64_t>> groups_processed_vec(concurrency);
- posting_t::block_intersector_t(
- posting_lists, iter_state, thread_pool, 100
- )
- .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its, size_t index) {
- score_results(sort_fields, searched_queries.size(), field_id, field_is_array,
- total_cost, topsters[index], query_suggestion, groups_processed_vec[index],
- seq_id, sort_order, field_values, geopoint_indices,
- group_limit, group_by_fields, token_bits,
- prioritize_exact_match, single_exact_query_token, its);
+ if(topster == nullptr) {
+ posting_t::block_intersector_t(
+ posting_lists, iter_state, thread_pool, 100
+ )
+ .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its, size_t index) {
+ result_id_vecs[index].push_back(seq_id);
+ }, concurrency);
+ } else {
+ for(size_t i = 0; i < concurrency; i++) {
+ topsters[i] = new Topster(topster->MAX_SIZE, topster->distinct);
+ }
- result_id_vecs[index].push_back(seq_id);
- }, concurrency);
+ posting_t::block_intersector_t(
+ posting_lists, iter_state, thread_pool, 100
+ )
+ .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its, size_t index) {
+ score_results(sort_fields, searched_queries.size(), field_id, field_is_array,
+ total_cost, topsters[index], query_suggestion, groups_processed_vec[index],
+ seq_id, sort_order, field_values, geopoint_indices,
+ group_limit, group_by_fields, token_bits,
+ prioritize_exact_match, single_exact_query_token, its);
+
+ result_id_vecs[index].push_back(seq_id);
+ }, concurrency);
+ }
delete [] excluded_result_ids;
size_t num_result_ids = 0;
for(size_t i = 0; i < concurrency; i++) {
- aggregate_topster(*topster, topsters[i]);
- delete topsters[i];
-
uint32_t* new_all_result_ids = nullptr;
all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &result_id_vecs[i][0],
result_id_vecs[i].size(), &new_all_result_ids);
@@ -1041,7 +1046,11 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
num_result_ids += result_id_vecs[i].size();
- groups_processed.insert(groups_processed_vec[i].begin(), groups_processed_vec[i].end());
+ if(topster != nullptr) {
+ aggregate_topster(*topster, topsters[i]);
+ delete topsters[i];
+ groups_processed.insert(groups_processed_vec[i].begin(), groups_processed_vec[i].end());
+ }
}
if(num_result_ids == 0) {
@@ -1400,6 +1409,7 @@ void Index::run_search(search_args* search_params) {
search_params->raw_result_kvs, search_params->override_result_kvs,
search_params->typo_tokens_threshold,
search_params->group_limit, search_params->group_by_fields,
+ search_params->dynamic_overrides,
search_params->default_sorting_field,
search_params->prioritize_exact_match,
search_params->exhaustive_search,
@@ -1421,6 +1431,10 @@ void Index::collate_included_ids(const std::vector<std::string>& q_included_toke
std::vector<art_leaf *> override_query;
for(const std::string& token: q_included_tokens) {
+ if(token == "*") {
+ continue;
+ }
+
const size_t token_len = token.size() + 1;
std::vector<art_leaf*> leaves;
@@ -1475,7 +1489,140 @@ void Index::concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t,
}
}
-void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
+void Index::process_dynamic_overrides(const std::vector<const override_t*>& dynamic_overrides,
+ std::vector<query_tokens_t>& field_query_tokens,
+ token_ordering token_order,
+ uint32_t** filter_ids,
+ uint32_t& filter_ids_length) const {
+
+ // compute dynamic filters if any
+ for(auto& override: dynamic_overrides) {
+ // will do an AND across different fields
+ uint32_t* field_override_ids = nullptr;
+ size_t field_override_ids_len = 0;
+
+ // check for dynamic filter
+ for(auto& field_name: override->dynamic_filters) {
+ // check if query matches a filter field
+ std::vector<std::string> tokens = field_query_tokens[0].q_include_tokens;
+ check_for_overrides(token_order, field_name, override->mutate_query_string,
+ field_override_ids, field_override_ids_len, tokens);
+
+ for(size_t i=0; i < field_query_tokens.size(); i++) {
+ if(tokens.empty()) {
+ field_query_tokens[i].q_include_tokens = {"*"};
+ } else {
+ field_query_tokens[i].q_include_tokens = tokens;
+ }
+ }
+
+ for(size_t i = 0; i < field_query_tokens[0].q_synonyms.size(); i++) {
+ uint32_t* synonym_override_ids = nullptr;
+ size_t synonym_override_ids_len = 0;
+
+ std::vector<std::string> syn_tokens = field_query_tokens[0].q_synonyms[i];
+ check_for_overrides(token_order, field_name, override->mutate_query_string,
+ synonym_override_ids, synonym_override_ids_len,syn_tokens);
+
+ for(size_t j=0; j < field_query_tokens.size(); j++) {
+ if(syn_tokens.empty()) {
+ field_query_tokens[j].q_synonyms[i] = {"*"};
+ } else {
+ field_query_tokens[j].q_synonyms[i] = syn_tokens;
+ }
+ }
+
+ // result ids of synonyms will be ORed
+
+ uint32_t* syn_field_override_ids = nullptr;
+ field_override_ids_len = ArrayUtils::or_scalar(field_override_ids, field_override_ids_len,
+ synonym_override_ids,
+ synonym_override_ids_len, &syn_field_override_ids);
+
+ delete [] synonym_override_ids;
+ delete [] field_override_ids;
+ field_override_ids = syn_field_override_ids;
+ }
+ }
+
+ if(filter_ids_length != 0) {
+ uint32_t* filtered_results = nullptr;
+ filter_ids_length = ArrayUtils::and_scalar(field_override_ids, field_override_ids_len, *filter_ids,
+ filter_ids_length, &filtered_results);
+ delete [] *filter_ids;
+ delete [] field_override_ids;
+ *filter_ids = filtered_results;
+ } else {
+ *filter_ids = field_override_ids;
+ filter_ids_length = field_override_ids_len;
+ }
+ }
+}
+
+void
+Index::check_for_overrides(const token_ordering& token_order, const string& field_name, bool mutate_query_string,
+ uint32_t*& field_override_ids,
+ size_t& field_override_ids_len, std::vector<std::string>& actual_tokens) const {
+
+ std::vector<std::string> local_tokens = actual_tokens;
+ std::vector<std::string>& tokens = mutate_query_string ? actual_tokens : local_tokens;
+
+ DO_WINDOWING:
+ for(size_t window_len = tokens.size(); window_len > 0; window_len--) {
+ for(size_t start_index = 0; start_index+window_len-1 < tokens.size(); start_index++) {
+ std::vector<token_t> window_tokens;
+ std::set<std::string> window_tokens_set;
+ for (size_t i = start_index; i < start_index + window_len; i++) {
+ window_tokens.push_back({i, tokens[i]});
+ window_tokens_set.emplace(tokens[i]);
+ }
+
+ std::vector<token_t> search_tokens = window_tokens;
+
+ std::vector<facet> facets;
+ std::vector<std::vector<art_leaf*>> searched_queries;
+ Topster* topster = nullptr;
+ spp::sparse_hash_set<uint64_t> groups_processed;
+ uint32_t* result_ids = nullptr;
+ size_t result_ids_len = 0;
+ size_t field_num_results = 0;
+ std::vector<std::string> group_by_fields;
+
+ size_t num_toks_dropped = 0;
+ search_field(0, window_tokens, search_tokens, nullptr, 0, num_toks_dropped, field_name,
+ nullptr, 0, {}, facets, {}, 2, searched_queries, topster, groups_processed,
+ &result_ids, result_ids_len, field_num_results, 0, group_by_fields,
+ false, 4, token_order, false, 0, 1, false);
+
+ if(result_ids_len != 0) {
+ if(field_override_ids != nullptr) {
+ uint32_t* filtered_results = nullptr;
+ field_override_ids_len = ArrayUtils::and_scalar(field_override_ids, field_override_ids_len, result_ids,
+ result_ids_len, &filtered_results);
+ delete [] result_ids;
+ delete [] field_override_ids;
+ field_override_ids = filtered_results;
+ } else {
+ field_override_ids = result_ids;
+ field_override_ids_len = result_ids_len;
+ }
+
+ // remove window_tokens from `tokens` and repeat search
+ std::vector<std::string> new_tokens;
+ for(const auto& token: tokens) {
+ if(window_tokens_set.count(token) == 0) {
+ new_tokens.emplace_back(token);
+ }
+ }
+
+ tokens = new_tokens;
+ goto DO_WINDOWING;
+ }
+ }
+ }
+}
+
+void Index::search(std::vector<query_tokens_t>& field_query_tokens,
const std::vector<search_field_t>& search_fields,
const std::vector<filter>& filters,
std::vector<facet>& facets, facet_query_t& facet_query,
@@ -1494,6 +1641,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
const size_t typo_tokens_threshold,
const size_t group_limit,
const std::vector<std::string>& group_by_fields,
+ const std::vector<const override_t*>& dynamic_overrides,
const std::string& default_sorting_field,
bool prioritize_exact_match,
const bool exhaustive_search,
@@ -1524,6 +1672,8 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
std::vector<uint32_t> curated_ids_sorted(curated_ids.begin(), curated_ids.end());
std::sort(curated_ids_sorted.begin(), curated_ids_sorted.end());
+ process_dynamic_overrides(dynamic_overrides, field_query_tokens, token_order, &filter_ids, filter_ids_length);
+
// Order of `fields` are used to sort results
//auto begin = std::chrono::high_resolution_clock::now();
uint32_t* all_result_ids = nullptr;
@@ -1568,70 +1718,14 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
const std::string& field = search_fields[0].name;
- // if a filter is not specified, use the seq_ids index to generate the list of all document ids
- if(filters.empty()) {
- filter_ids_length = seq_ids.getLength();
- filter_ids = seq_ids.uncompress();
- }
-
- if(!curated_ids.empty()) {
- uint32_t *excluded_result_ids = nullptr;
- filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, &curated_ids_sorted[0],
- curated_ids_sorted.size(), &excluded_result_ids);
- delete [] filter_ids;
- filter_ids = excluded_result_ids;
- }
+ curate_filtered_ids(filters, curated_ids, exclude_token_ids,
+ exclude_token_ids_size, filter_ids, filter_ids_length, curated_ids_sorted);
- // Exclude document IDs associated with excluded tokens from the result set
- if(exclude_token_ids_size != 0) {
- uint32_t *excluded_result_ids = nullptr;
- filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, exclude_token_ids,
- exclude_token_ids_size, &excluded_result_ids);
- delete[] filter_ids;
- filter_ids = excluded_result_ids;
- }
-
-
- // FIXME: duplicated
- int sort_order[3]; // 1 or -1 based on DESC or ASC respectively
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values;
- std::vector<size_t> geopoint_indices;
-
- for (size_t i = 0; i < sort_fields_std.size(); i++) {
- sort_order[i] = 1;
- if (sort_fields_std[i].order == sort_field_const::asc) {
- sort_order[i] = -1;
- }
-
- if (sort_fields_std[i].name == sort_field_const::text_match) {
- field_values[i] = &text_match_sentinel_value;
- } else if (sort_fields_std[i].name == sort_field_const::seq_id) {
- field_values[i] = &seq_id_sentinel_value;
- } else if (sort_index.count(sort_fields_std[i].name) != 0) {
- field_values[i] = sort_index.at(sort_fields_std[i].name);
-
- if (sort_schema.at(sort_fields_std[i].name).is_geopoint()) {
- geopoint_indices.push_back(i);
- }
- }
- }
-
- uint32_t token_bits = 255;
- std::vector<posting_list_t::iterator_t> plists;
-
- for(size_t i = 0; i < filter_ids_length; i++) {
- const uint32_t seq_id = filter_ids[i];
- score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, false, 0, topster, {},
- groups_processed, seq_id, sort_order, field_values,
- geopoint_indices, group_limit, group_by_fields, token_bits,
- false, false, plists);
- }
-
- collate_included_ids(field_query_tokens[0].q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);
-
- all_result_ids_len = filter_ids_length;
- all_result_ids = filter_ids;
- filter_ids = nullptr;
+ search_wildcard(field_query_tokens[0].q_include_tokens, filters, included_ids_map, sort_fields_std, topster,
+ curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,
+ curated_ids, curated_ids_sorted,
+ exclude_token_ids, exclude_token_ids_size, field_id, field,
+ all_result_ids, all_result_ids_len, filter_ids, filter_ids_length);
} else {
// In multi-field searches, a record can be matched across different fields, so we use this for aggregation
spp::sparse_hash_map<uint64_t, std::vector<KV*>> topster_ids;
@@ -1685,18 +1779,37 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
token_order, field_prefix,
drop_tokens_threshold, typo_tokens_threshold, exhaustive_search);
+ bool syn_wildcard_filter_init_done = false;
+
// do synonym based searches
for(const auto& syn_tokens: q_pos_synonyms) {
num_tokens_dropped = 0;
field_num_results = 0;
query_tokens = search_tokens = syn_tokens;
- search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,
- field_name, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
- field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,
- field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,
- token_order, field_prefix,
- drop_tokens_threshold, typo_tokens_threshold, exhaustive_search);
+ if(query_tokens.size() == 1 && query_tokens[0].value == "*") {
+ // synonym can be a wildcard if there was an override filter used
+ // we will only do filtering in that case
+
+ if(!syn_wildcard_filter_init_done) {
+ curate_filtered_ids(filters, curated_ids, exclude_token_ids, exclude_token_ids_size,
+ filter_ids, filter_ids_length, curated_ids_sorted);
+ syn_wildcard_filter_init_done = true;
+ }
+
+ search_wildcard({"*"}, filters, included_ids_map, sort_fields_std, actual_topster, curated_topster,
+ groups_processed, searched_queries, group_limit, group_by_fields,
+ curated_ids, curated_ids_sorted,
+ exclude_token_ids, exclude_token_ids_size, field_id, field_name,
+ all_result_ids, all_result_ids_len, filter_ids, filter_ids_length);
+ } else {
+ search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,
+ field_name, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
+ field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,
+ field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,
+ token_order, field_prefix,
+ drop_tokens_threshold, typo_tokens_threshold, exhaustive_search);
+ }
}
// concat is done only for multi-field searches as `ftopster` will be empty for single-field search
@@ -1988,6 +2101,90 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
//LOG(INFO) << "Time taken for result calc: " << timeMillis << "ms";
}
+void Index::curate_filtered_ids(const std::vector<filter>& filters, const std::set<uint32_t>& curated_ids,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
+ uint32_t*& filter_ids, uint32_t& filter_ids_length,
+ const std::vector<uint32_t>& curated_ids_sorted) const {
+ // if filtered results are not available, use the seq_ids index to generate the list of all document ids
+
+ if(filters.empty() && filter_ids_length == 0) {
+ filter_ids_length = seq_ids.getLength();
+ filter_ids = seq_ids.uncompress();
+ }
+
+ if(!curated_ids.empty()) {
+ uint32_t *excluded_result_ids = nullptr;
+ filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, &curated_ids_sorted[0],
+ curated_ids_sorted.size(), &excluded_result_ids);
+ delete [] filter_ids;
+ filter_ids = excluded_result_ids;
+ }
+
+ // Exclude document IDs associated with excluded tokens from the result set
+ if(exclude_token_ids_size != 0) {
+ uint32_t *excluded_result_ids = nullptr;
+ filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, exclude_token_ids,
+ exclude_token_ids_size, &excluded_result_ids);
+ delete[] filter_ids;
+ filter_ids = excluded_result_ids;
+ }
+}
+
+void Index::search_wildcard(const std::vector<std::string>& qtokens, const std::vector<filter>& filters,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ const std::vector<sort_by>& sort_fields_std, Topster* topster, Topster* curated_topster,
+ spp::sparse_hash_set<uint64_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
+ const std::set<uint32_t>& curated_ids, const std::vector<uint32_t>& curated_ids_sorted,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size, const uint8_t field_id,
+ const string& field, uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const uint32_t* filter_ids, uint32_t filter_ids_length) const {
+
+ // FIXME: duplicated
+ int sort_order[3]; // 1 or -1 based on DESC or ASC respectively
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values;
+ std::vector<size_t> geopoint_indices;
+
+ for (size_t i = 0; i < sort_fields_std.size(); i++) {
+ sort_order[i] = 1;
+ if (sort_fields_std[i].order == sort_field_const::asc) {
+ sort_order[i] = -1;
+ }
+
+ if (sort_fields_std[i].name == sort_field_const::text_match) {
+ field_values[i] = &text_match_sentinel_value;
+ } else if (sort_fields_std[i].name == sort_field_const::seq_id) {
+ field_values[i] = &seq_id_sentinel_value;
+ } else if (sort_index.count(sort_fields_std[i].name) != 0) {
+ field_values[i] = sort_index.at(sort_fields_std[i].name);
+
+ if (sort_schema.at(sort_fields_std[i].name).is_geopoint()) {
+ geopoint_indices.push_back(i);
+ }
+ }
+ }
+
+ uint32_t token_bits = 255;
+ std::vector<posting_list_t::iterator_t> plists;
+
+ for(size_t i = 0; i < filter_ids_length; i++) {
+ const uint32_t seq_id = filter_ids[i];
+ score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, false, 0, topster, {},
+ groups_processed, seq_id, sort_order, field_values,
+ geopoint_indices, group_limit, group_by_fields, token_bits,
+ false, false, plists);
+ }
+
+ collate_included_ids(qtokens, field, field_id, included_ids_map, curated_topster, searched_queries);
+
+ uint32_t* new_all_result_ids = nullptr;
+ all_result_ids_len = ArrayUtils::or_scalar(all_result_ids, all_result_ids_len, filter_ids,
+ filter_ids_length, &new_all_result_ids);
+ delete [] all_result_ids;
+ all_result_ids = new_all_result_ids;
+}
+
/*
1. Split the query into tokens
2. Outer loop will generate bounded cartesian product with costs for each token
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index ca33b70d..eff9f8a8 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -262,7 +262,7 @@ TEST_F(CollectionOverrideTest, OverrideJSONValidation) {
parse_op = override_t::parse(include_json2, "", override2);
ASSERT_FALSE(parse_op.ok());
- ASSERT_STREQ("Must contain either `includes` or `excludes`.", parse_op.error().c_str());
+ ASSERT_STREQ("Must contain one of:`includes`, `excludes`, `dynamic_filters`.", parse_op.error().c_str());
include_json2["includes"] = nlohmann::json::array();
include_json2["includes"][0] = 100;
@@ -788,4 +788,386 @@ TEST_F(CollectionOverrideTest, PinnedHitsIdsHavingColon) {
ASSERT_STREQ("https://example.com/2", res["hits"][9]["document"]["id"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
-}
\ No newline at end of file
+}
+
+TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("category", field_types::STRING, true),
+ field("brand", field_types::STRING, true),
+ field("points", field_types::INT32, false)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Amazing Shoes";
+ doc1["category"] = "shoes";
+ doc1["brand"] = "Nike";
+ doc1["points"] = 3;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "Track Gym";
+ doc2["category"] = "shoes";
+ doc2["brand"] = "Adidas";
+ doc2["points"] = 5;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+
+ std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
+
+ auto results = coll1->search("shoes", {"name", "category", "brand"}, "",
+ {}, sort_fields, {2, 2, 2}, 10).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+
+ // with override, results will be different
+
+ nlohmann::json override_json = {
+ {"id", "dynamic-filters"},
+ {
+ "rule", {
+ {"query", ".*"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ },
+ {"mutate_query_string", true},
+ {"dynamic_filters", "category,brand"}
+ };
+
+ override_t override;
+ auto op = override_t::parse(override_json, "dynamic-filters", override);
+ ASSERT_TRUE(op.ok());
+
+ coll1->add_override(override);
+
+ results = coll1->search("shoes", {"name", "category", "brand"}, "",
+ {}, sort_fields, {2, 2, 2}, 10).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+
+ results = coll1->search("adidas shoes", {"name", "category", "brand"}, "",
+ {}, sort_fields, {2, 2, 2}, 10).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+
+ // with bad override
+
+ nlohmann::json override_json_bad1 = {
+ {"id", "dynamic-filters-bad1"},
+ {
+ "rule", {
+ {"query", ".*"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ },
+ {"mutate_query_string", true},
+ {"dynamic_filters", ""}
+ };
+
+ override_t override_bad1;
+ op = override_t::parse(override_json_bad1, "dynamic-filters-bad1", override_bad1);
+ ASSERT_FALSE(op.ok());
+ ASSERT_EQ("The `dynamic_filters` must be a comma separated string of filter field names.", op.error());
+
+ nlohmann::json override_json_bad2 = {
+ {"id", "dynamic-filters-bad2"},
+ {
+ "rule", {
+ {"query", ".*"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ },
+ {"mutate_query_string", true},
+ {"dynamic_filters", {"foo", "bar"}}
+ };
+
+ override_t override_bad2;
+ op = override_t::parse(override_json_bad2, "dynamic-filters-bad2", override_bad2);
+ ASSERT_FALSE(op.ok());
+ ASSERT_EQ("The `dynamic_filters` must be a comma separated string of filter field names.", op.error());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionOverrideTest, DynamicFilteringWithSynonyms) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("category", field_types::STRING, true),
+ field("brand", field_types::STRING, true),
+ field("points", field_types::INT32, false)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Amazing Shoes";
+ doc1["category"] = "shoes";
+ doc1["brand"] = "Nike";
+ doc1["points"] = 3;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "Track Gym";
+ doc2["category"] = "shoes";
+ doc2["brand"] = "Adidas";
+ doc2["points"] = 5;
+
+ nlohmann::json doc3;
+ doc3["id"] = "2";
+ doc3["name"] = "Amazing Sneakers";
+ doc3["category"] = "sneakers";
+ doc3["brand"] = "Adidas";
+ doc3["points"] = 4;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());
+
+ synonym_t synonym1{"sneakers-shoes", {"sneakers"}, {{"shoes"}} };
+ synonym_t synonym2{"boots-shoes", {"boots"}, {{"shoes"}} };
+ coll1->add_synonym(synonym1);
+ coll1->add_synonym(synonym2);
+
+ std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
+
+ // with override, results will be different
+
+ nlohmann::json override_json = {
+ {"id", "dynamic-filters"},
+ {
+ "rule", {
+ {"query", ".*"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ },
+ {"mutate_query_string", true},
+ {"dynamic_filters", "category,brand"}
+ };
+
+ override_t override;
+ auto op = override_t::parse(override_json, "dynamic-filters", override);
+ ASSERT_TRUE(op.ok());
+
+ coll1->add_override(override);
+
+ auto results = coll1->search("sneakers", {"name", "category", "brand"}, "",
+ {}, sort_fields, {2, 2, 2}, 10).get();
+
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][2]["document"]["id"].get<std::string>());
+
+ // keyword does not exist but has a synonym with results
+
+ results = coll1->search("boots", {"name", "category", "brand"}, "",
+ {}, sort_fields, {2, 2, 2}, 10).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionOverrideTest, StaticFiltering) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("price", field_types::FLOAT, true),
+ field("points", field_types::INT32, false)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Amazing Shoes";
+ doc1["price"] = 399.99;
+ doc1["points"] = 3;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "Track Shoes";
+ doc2["price"] = 49.99;
+ doc2["points"] = 5;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+
+ std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
+
+ nlohmann::json override_json_contains = {
+ {"id", "static-filters"},
+ {
+ "rule", {
+ {"query", "expensive"},
+ {"match", override_t::MATCH_CONTAINS}
+ }
+ },
+ {"mutate_query_string", true},
+ {"dynamic_filters", "price:> 100"}
+ };
+
+ override_t override_contains;
+ auto op = override_t::parse(override_json_contains, "static-filters", override_contains);
+ ASSERT_TRUE(op.ok());
+
+ coll1->add_override(override_contains);
+
+ nlohmann::json override_json_exact = {
+ {"id", "static-exact-filters"},
+ {
+ "rule", {
+ {"query", "cheap"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ },
+ {"mutate_query_string", true},
+ {"dynamic_filters", "price:< 100"}
+ };
+
+ override_t override_exact;
+ op = override_t::parse(override_json_exact, "static-exact-filters", override_exact);
+ ASSERT_TRUE(op.ok());
+
+ coll1->add_override(override_exact);
+
+ auto results = coll1->search("expensive shoes", {"name"}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+
+ results = coll1->search("expensive", {"name"}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+
+ // with exact match
+
+ results = coll1->search("cheap", {"name"}, "",
+ {}, sort_fields, {2}, 10).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+
+ // should not work in match contains context
+
+ results = coll1->search("cheap boots", {"name"}, "",
+ {}, sort_fields, {2}, 10).get();
+
+ ASSERT_EQ(0, results["hits"].size());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionOverrideTest, FilteringWithAndWithoutQueryStringMutation) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("price", field_types::FLOAT, true),
+ field("points", field_types::INT32, false)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Apple iPad";
+ doc1["price"] = 399.99;
+ doc1["points"] = 3;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "Samsung Charger";
+ doc2["price"] = 49.99;
+ doc2["points"] = 5;
+
+ nlohmann::json doc3;
+ doc3["id"] = "2";
+ doc3["name"] = "Samsung Phone";
+ doc3["price"] = 249.99;
+ doc3["points"] = 5;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());
+
+ std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
+
+ nlohmann::json override_json_contains = {
+ {"id", "static-filters"},
+ {
+ "rule", {
+ {"query", "apple"},
+ {"match", override_t::MATCH_CONTAINS}
+ }
+ },
+ {"mutate_query_string", false},
+ {"dynamic_filters", "price:> 200"}
+ };
+
+ override_t override_contains;
+ auto op = override_t::parse(override_json_contains, "static-filters", override_contains);
+ ASSERT_TRUE(op.ok());
+
+ coll1->add_override(override_contains);
+
+ // first without query string mutation
+
+ auto results = coll1->search("apple", {"name"}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+
+ // now, with query string mutation
+
+ override_json_contains = {
+ {"id", "static-filters"},
+ {
+ "rule", {
+ {"query", "apple"},
+ {"match", override_t::MATCH_CONTAINS}
+ }
+ },
+ {"mutate_query_string", true},
+ {"dynamic_filters", "price:> 200"}
+ };
+
+ op = override_t::parse(override_json_contains, "static-filters", override_contains);
+ ASSERT_TRUE(op.ok());
+ coll1->add_override(override_contains);
+
+ results = coll1->search("apple", {"name"}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}