Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Filter query (#737)

Harpreet Sangarf75cbca92044
diff --git a/include/art.h b/include/art.h
index 0235bf82..0502641c 100644
--- a/include/art.h
+++ b/include/art.h
@@ -157,6 +157,11 @@ enum NUM_COMPARATOR {
RANGE_INCLUSIVE
};
+enum FILTER_OPERATOR {
+ AND,
+ OR
+};
+
/**
* Initializes an ART tree
* @return 0 on success.
diff --git a/include/collection.h b/include/collection.h
index 1c1b0aae..7f1a4b83 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -212,7 +212,7 @@ private:
void process_filter_overrides(std::vector<const override_t*>& filter_overrides,
std::vector<std::string>& q_include_tokens,
token_ordering token_order,
- std::vector<filter>& filters,
+ filter_node_t*& filter_tree_root,
std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
std::vector<uint32_t>& excluded_ids) const;
@@ -379,7 +379,7 @@ public:
const bool& return_doc=false, const bool& return_id=false);
Option<nlohmann::json> search(const std::string & query, const std::vector<std::string> & search_fields,
- const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
+ const std::string & filter_query, const std::vector<std::string> & facet_fields,
const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,
size_t per_page = 10, size_t page = 1,
token_ordering token_order = FREQUENCY, const std::vector<bool>& prefixes = {true},
diff --git a/include/field.h b/include/field.h
index da5ae568..bff58b4b 100644
--- a/include/field.h
+++ b/include/field.h
@@ -123,7 +123,7 @@ struct field {
bool is_integer() const {
return (type == field_types::INT32 || type == field_types::INT32_ARRAY ||
- type == field_types::INT64 || type == field_types::INT64_ARRAY);
+ type == field_types::INT64 || type == field_types::INT64_ARRAY);
}
bool is_int32() const {
@@ -175,7 +175,7 @@ struct field {
static bool is_dynamic(const std::string& name, const std::string& type) {
return type == "string*" || (name != ".*" && type == field_types::AUTO) ||
- (name != ".*" && name.find(".*") != std::string::npos);
+ (name != ".*" && name.find(".*") != std::string::npos);
}
bool is_dynamic() const {
@@ -307,19 +307,19 @@ struct field {
if(!field.has_valid_type()) {
return Option<bool>(400, "Field `" + field.name +
- "` has an invalid data type `" + field.type +
- "`, see docs for supported data types.");
+ "` has an invalid data type `" + field.type +
+ "`, see docs for supported data types.");
}
if(field.name == default_sorting_field && !field.is_sortable()) {
return Option<bool>(400, "Default sorting field `" + default_sorting_field +
- "` is not a sortable type.");
+ "` is not a sortable type.");
}
if(field.name == default_sorting_field) {
if(field.optional) {
return Option<bool>(400, "Default sorting field `" + default_sorting_field +
- "` cannot be an optional field.");
+ "` cannot be an optional field.");
}
if(field.is_geopoint()) {
@@ -353,7 +353,7 @@ struct field {
if(!default_sorting_field.empty() && !found_default_sorting_field && !fields.empty()) {
return Option<bool>(400, "Default sorting field is defined as `" + default_sorting_field +
- "` but is not found in the schema.");
+ "` but is not found in the schema.");
}
// check for duplicate field names in schema
@@ -422,6 +422,8 @@ struct field {
static Option<bool> flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields);
};
+struct filter_node_t;
+
struct filter {
std::string field_name;
std::vector<std::string> values;
@@ -454,7 +456,7 @@ struct filter {
num_comparator = EQUALS;
}
- // the ordering is important - we have to compare 2-letter operators first
+ // the ordering is important - we have to compare 2-letter operators first
else if(comp_and_value.compare(0, 2, "<=") == 0) {
num_comparator = LESS_THAN_EQUALS;
}
@@ -499,7 +501,34 @@ struct filter {
const tsl::htrie_map<char, field>& search_schema,
const Store* store,
const std::string& doc_id_prefix,
- std::vector<filter>& filters);
+ filter_node_t*& root);
+};
+
+struct filter_node_t {
+ filter filter_exp;
+ FILTER_OPERATOR filter_operator;
+ bool isOperator;
+ filter_node_t* left;
+ filter_node_t* right;
+
+ filter_node_t(filter filter_exp)
+ : filter_exp(std::move(filter_exp)),
+ isOperator(false),
+ left(nullptr),
+ right(nullptr) {}
+
+ filter_node_t(FILTER_OPERATOR filter_operator,
+ filter_node_t* left,
+ filter_node_t* right)
+ : filter_operator(filter_operator),
+ isOperator(true),
+ left(left),
+ right(right) {}
+
+ ~filter_node_t() {
+ delete left;
+ delete right;
+ }
};
namespace sort_field_const {
@@ -526,7 +555,7 @@ struct sort_by {
};
struct eval_t {
- std::vector<filter> filters;
+ filter_node_t* filter_tree_root;
uint32_t* ids = nullptr;
uint32_t size = 0;
};
@@ -546,8 +575,8 @@ struct sort_by {
eval_t eval;
sort_by(const std::string & name, const std::string & order):
- name(name), order(order), text_match_buckets(0), geopoint(0), exclude_radius(0), geo_precision(0),
- missing_values(normal) {
+ name(name), order(order), text_match_buckets(0), geopoint(0), exclude_radius(0), geo_precision(0),
+ missing_values(normal) {
}
diff --git a/include/index.h b/include/index.h
index 473571bb..042b9c61 100644
--- a/include/index.h
+++ b/include/index.h
@@ -90,7 +90,7 @@ enum enable_t {
struct search_args {
std::vector<query_tokens_t> field_query_tokens;
std::vector<search_field_t> search_fields;
- std::vector<filter> filters;
+ const filter_node_t* filter_tree_root;
std::vector<facet>& facets;
std::vector<std::pair<uint32_t, uint32_t>>& included_ids;
std::vector<uint32_t> excluded_ids;
@@ -134,7 +134,7 @@ struct search_args {
vector_query_t& vector_query;
search_args(std::vector<query_tokens_t> field_query_tokens, std::vector<search_field_t> search_fields,
- std::vector<filter> filters, std::vector<facet>& facets,
+ filter_node_t* filter_tree_root, std::vector<facet>& facets,
std::vector<std::pair<uint32_t, uint32_t>>& included_ids, std::vector<uint32_t> excluded_ids,
std::vector<sort_by>& sort_fields_std, facet_query_t facet_query, const std::vector<uint32_t>& num_typos,
size_t max_facet_values, size_t max_hits, size_t per_page, size_t page, token_ordering token_order,
@@ -147,7 +147,7 @@ struct search_args {
const size_t max_extra_prefix, const size_t max_extra_suffix, const size_t facet_query_num_typos,
const bool filter_curated_hits, const enable_t split_join_tokens, vector_query_t& vector_query) :
field_query_tokens(field_query_tokens),
- search_fields(search_fields), filters(filters), facets(facets),
+ search_fields(search_fields), filter_tree_root(filter_tree_root), facets(facets),
included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),
facet_query(facet_query), num_typos(num_typos), max_facet_values(max_facet_values), per_page(per_page),
page(page), token_order(token_order), prefixes(prefixes),
@@ -362,7 +362,7 @@ private:
const uint32_t* result_ids, size_t results_size) const;
bool static_filter_query_eval(const override_t* override, std::vector<std::string>& tokens,
- std::vector<filter>& filters) const;
+ filter_node_t*& filter_tree_root) const;
bool resolve_override(const std::vector<std::string>& rule_tokens, bool exact_rule_match,
const std::vector<std::string>& query_tokens,
@@ -460,8 +460,14 @@ private:
const size_t num_search_fields,
std::vector<size_t>& popular_field_ids);
- void do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length, const std::vector<filter>& filters,
- const bool enable_short_circuit) const;
+ void do_filtering(uint32_t*& filter_ids,
+ uint32_t& filter_ids_length,
+ filter_node_t const* const root) const;
+
+ void recursive_filter(uint32_t*& filter_ids,
+ uint32_t& filter_ids_length,
+ filter_node_t const* const root,
+ const bool enable_short_circuit) const;
void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;
@@ -616,7 +622,7 @@ public:
void run_search(search_args* search_params);
void search(std::vector<query_tokens_t>& field_query_tokens, const std::vector<search_field_t>& the_fields,
- std::vector<filter>& filters, std::vector<facet>& facets, facet_query_t& facet_query,
+ filter_node_t const* const& filter_tree_root, std::vector<facet>& facets, facet_query_t& facet_query,
const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
const std::vector<uint32_t>& excluded_ids, std::vector<sort_by>& sort_fields_std,
const std::vector<uint32_t>& num_typos, Topster* topster, Topster* curated_topster,
@@ -673,8 +679,10 @@ public:
art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);
- void do_filtering_with_lock(uint32_t*& filter_ids, uint32_t& filter_ids_length,
- const std::vector<filter>& filters) const;
+ void do_filtering_with_lock(
+ uint32_t*& filter_ids,
+ uint32_t& filter_ids_length,
+ filter_node_t const* const& filter_tree_root) const;
void refresh_schemas(const std::vector<field>& new_fields, const std::vector<field>& del_fields);
@@ -687,7 +695,7 @@ public:
const std::string& fallback_field_type,
const DIRTY_VALUES& dirty_values);
- void search_wildcard(const std::vector<filter>& filters,
+ void search_wildcard(filter_node_t const* const& filter_tree_root,
const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
spp::sparse_hash_set<uint64_t>& groups_processed,
@@ -704,7 +712,7 @@ public:
void search_infix(const std::string& query, const std::string& field_name, std::vector<uint32_t>& ids,
size_t max_extra_prefix, size_t max_extra_suffix) const;
- void curate_filtered_ids(const std::vector<filter>& filters, const std::set<uint32_t>& curated_ids,
+ void curate_filtered_ids(filter_node_t const* const& filter_tree_root, const std::set<uint32_t>& curated_ids,
const uint32_t* exclude_token_ids, size_t exclude_token_ids_size, uint32_t*& filter_ids,
uint32_t& filter_ids_length, const std::vector<uint32_t>& curated_ids_sorted) const;
@@ -745,7 +753,7 @@ public:
spp::sparse_hash_set<uint64_t>& groups_processed) const;
void do_synonym_search(const std::vector<search_field_t>& the_fields,
- const std::vector<filter>& filters,
+ filter_node_t const* const& filter_tree_root,
const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
const token_ordering& token_order,
@@ -865,7 +873,7 @@ public:
void process_filter_overrides(const std::vector<const override_t*>& filter_overrides,
std::vector<std::string>& query_tokens,
token_ordering token_order,
- std::vector<filter>& filters,
+ filter_node_t*& filter_tree_root,
std::vector<const override_t*>& matched_dynamic_overrides) const;
void compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,
diff --git a/include/string_utils.h b/include/string_utils.h
index 43e4fbcd..5a793b65 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -7,8 +7,10 @@
#include <vector>
#include <random>
#include <map>
+#include <queue>
#include "wyhash_v5.h"
#include <unicode/normalizer2.h>
+#include "option.h"
struct StringUtils {
@@ -21,8 +23,8 @@ struct StringUtils {
// Adapted from: http://stackoverflow.com/a/236180/131050
static size_t split(const std::string& s, std::vector<std::string> & result, const std::string& delim,
- const bool keep_empty = false, const size_t start_index = 0,
- const size_t max_values = (std::numeric_limits<size_t>::max()-1)) {
+ const bool keep_empty = false, const size_t start_index = 0,
+ const size_t max_values = (std::numeric_limits<size_t>::max()-1)) {
if (delim.empty()) {
result.push_back(s);
return s.size();
@@ -319,4 +321,6 @@ struct StringUtils {
static char* get_ip_str(const struct sockaddr* sa, char* s, size_t maxlen);
static size_t get_num_chars(const std::string& text);
-};
\ No newline at end of file
+
+ static Option<bool> tokenize(std::string filter_query, std::queue<std::string>& tokens);
+};
diff --git a/src/collection.cpp b/src/collection.cpp
index 6e8fd9d3..68122e9f 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -586,7 +586,7 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<
sort_field_std.name.size() - paran_start -
2);
Option<bool> parse_filter_op = filter::parse_filter_query(filter_exp, search_schema,
- store, "", sort_field_std.eval.filters);
+ store, "", sort_field_std.eval.filter_tree_root);
if(!parse_filter_op.ok()) {
return Option<bool>(parse_filter_op.code(), "Error parsing eval expression in sort_by clause.");
}
@@ -825,7 +825,7 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,
Option<nlohmann::json> Collection::search(const std::string & raw_query,
const std::vector<std::string>& raw_search_fields,
- const std::string & simple_filter_query, const std::vector<std::string>& facet_fields,
+ const std::string & filter_query, const std::vector<std::string>& facet_fields,
const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,
const size_t per_page, const size_t page,
token_ordering token_order, const std::vector<bool>& prefixes,
@@ -1029,9 +1029,9 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
std::vector<facet> facets;
const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";
- std::vector<filter> filters;
- Option<bool> parse_filter_op = filter::parse_filter_query(simple_filter_query, search_schema,
- store, doc_id_prefix, filters);
+ filter_node_t* filter_tree_root = nullptr;
+ Option<bool> parse_filter_op = filter::parse_filter_query(filter_query, search_schema,
+ store, doc_id_prefix, filter_tree_root);
if(!parse_filter_op.ok()) {
return Option<nlohmann::json>(parse_filter_op.code(), parse_filter_op.error());
}
@@ -1143,7 +1143,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
std::string query = raw_query;
bool filter_curated_hits = false;
std::string curated_sort_by;
- curate_results(query, simple_filter_query, enable_overrides, pre_segmented_query, pinned_hits, hidden_hits,
+ curate_results(query, filter_query, enable_overrides, pre_segmented_query, pinned_hits, hidden_hits,
included_ids, excluded_ids, filter_overrides, filter_curated_hits, curated_sort_by);
if(filter_curated_hits_option == 0 || filter_curated_hits_option == 1) {
@@ -1242,7 +1242,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
// process filter overrides first, before synonyms (order is important)
// included_ids, excluded_ids
- process_filter_overrides(filter_overrides, q_include_tokens, token_order, filters,
+ process_filter_overrides(filter_overrides, q_include_tokens, token_order, filter_tree_root,
included_ids, excluded_ids);
for(size_t i = 0; i < q_include_tokens.size(); i++) {
@@ -1268,7 +1268,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
size_t index_id = 0;
search_args* search_params = new search_args(field_query_tokens, weighted_search_fields,
- filters, facets, included_ids, excluded_ids,
+ filter_tree_root, facets, included_ids, excluded_ids,
sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
per_page, page, token_order, prefixes,
drop_tokens_threshold, typo_tokens_threshold,
@@ -1796,9 +1796,11 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
// free search params
delete search_params;
+ delete filter_tree_root;
+
result["search_cutoff"] = search_cutoff;
- result["request_params"] = nlohmann::json::object();;
+ result["request_params"] = nlohmann::json::object();
result["request_params"]["collection_name"] = name;
result["request_params"]["per_page"] = per_page;
result["request_params"]["q"] = query;
@@ -2022,13 +2024,13 @@ void Collection::process_highlight_fields(const std::vector<search_field_t>& sea
void Collection::process_filter_overrides(std::vector<const override_t*>& filter_overrides,
std::vector<std::string>& q_include_tokens,
token_ordering token_order,
- std::vector<filter>& filters,
+ filter_node_t*& filter_tree_root,
std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
std::vector<uint32_t>& excluded_ids) const {
std::vector<const override_t*> matched_dynamic_overrides;
index->process_filter_overrides(filter_overrides, q_include_tokens, token_order,
- filters, matched_dynamic_overrides);
+ filter_tree_root, matched_dynamic_overrides);
// we will check the dynamic overrides to see if they also have include/exclude
std::set<uint32_t> excluded_set;
@@ -2192,9 +2194,9 @@ Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,
std::shared_lock lock(mutex);
const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";
- std::vector<filter> filters;
+ filter_node_t* filter_tree_root = nullptr;
Option<bool> filter_op = filter::parse_filter_query(simple_filter_query, search_schema,
- store, doc_id_prefix, filters);
+ store, doc_id_prefix, filter_tree_root);
if(!filter_op.ok()) {
return filter_op;
@@ -2202,9 +2204,10 @@ Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,
uint32_t* filter_ids = nullptr;
uint32_t filter_ids_len = 0;
- index->do_filtering_with_lock(filter_ids, filter_ids_len, filters);
+ index->do_filtering_with_lock(filter_ids, filter_ids_len, filter_tree_root);
index_ids.emplace_back(filter_ids_len, filter_ids);
+ delete filter_tree_root;
return Option<bool>(true);
}
diff --git a/src/field.cpp b/src/field.cpp
index 3c57fda9..f9610a23 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -1,6 +1,7 @@
#include <store.h>
#include "field.h"
#include "magic_enum.hpp"
+#include <stack>
Option<bool> filter::parse_geopoint_filter_value(std::string& raw_value,
const std::string& format_err_msg,
@@ -73,298 +74,365 @@ Option<bool> filter::parse_geopoint_filter_value(std::string& raw_value,
return Option<bool>(true);
}
-Option<bool> filter::parse_filter_query(const string& simple_filter_query,
- const tsl::htrie_map<char, field>& search_schema,
- const Store* store,
- const std::string& doc_id_prefix,
- std::vector<filter>& filters) {
-
- std::vector<filter> exclude_filters; // to ensure that they go last in the list of filters
-
- std::vector<std::string> filter_blocks;
- StringUtils::split(simple_filter_query, filter_blocks, "&&");
-
- for(const std::string & filter_block: filter_blocks) {
- // split into [field_name, value]
- size_t found_index = filter_block.find(':');
-
- if(found_index == std::string::npos) {
- return Option<bool>(400, "Could not parse the filter query.");
- }
-
- std::string&& field_name = filter_block.substr(0, found_index);
- StringUtils::trim(field_name);
-
- if(field_name == "id") {
- filter id_filter;
- std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);
- StringUtils::trim(raw_value);
- std::string empty_filter_err = "Error with filter field `id`: Filter value cannot be empty.";
-
- if(raw_value.empty()) {
- return Option<bool>(400, empty_filter_err);
- }
-
- id_filter = {field_name, {}, {}};
+bool isOperator(const std::string& expression) {
+ return expression == "&&" || expression == "||";
+}
- NUM_COMPARATOR id_comparator = EQUALS;
- size_t filter_value_index = 0;
+// https://en.wikipedia.org/wiki/Shunting_yard_algorithm
+Option<bool> toPostfix(std::queue<std::string>& tokens, std::queue<std::string>& postfix) {
+ std::stack<std::string> operatorStack;
- if(raw_value[0] == '=') {
- id_comparator = EQUALS;
- while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
- } else if(raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
- return Option<bool>(400, "Not equals filtering is not supported on the `id` field.");
- }
+ while (!tokens.empty()) {
+ auto expression = tokens.front();
+ tokens.pop();
- if(filter_value_index != 0) {
- raw_value = raw_value.substr(filter_value_index);
+ if (isOperator(expression)) {
+ // We only have two operators &&, || having the same precedence and both being left associative.
+ while (!operatorStack.empty() && operatorStack.top() != "(") {
+ postfix.push(operatorStack.top());
+ operatorStack.pop();
}
- if(raw_value.empty()) {
- return Option<bool>(400, empty_filter_err);
+ operatorStack.push(expression);
+ } else if (expression == "(") {
+ operatorStack.push(expression);
+ } else if (expression == ")") {
+ while (!operatorStack.empty() && operatorStack.top() != "(") {
+ postfix.push(operatorStack.top());
+ operatorStack.pop();
}
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> doc_ids;
- StringUtils::split_to_values(raw_value.substr(1, raw_value.size() - 2), doc_ids);
-
- for(std::string& doc_id: doc_ids) {
- // we have to convert the doc_id to seq id
- std::string seq_id_str;
- StoreStatus seq_id_status = store->get(doc_id_prefix + doc_id, seq_id_str);
-
- if(seq_id_status != StoreStatus::FOUND) {
- continue;
- }
-
- id_filter.values.push_back(seq_id_str);
- id_filter.comparators.push_back(id_comparator);
- }
- } else {
- std::vector<std::string> doc_ids;
- StringUtils::split_to_values(raw_value, doc_ids); // to handle backticks
-
- std::string seq_id_str;
- StoreStatus seq_id_status = store->get(doc_id_prefix + doc_ids[0], seq_id_str);
- if(seq_id_status == StoreStatus::FOUND) {
- id_filter.values.push_back(seq_id_str);
- id_filter.comparators.push_back(id_comparator);
- }
+ if (operatorStack.empty() || operatorStack.top() != "(") {
+ return Option<bool>(400, "Could not parse the filter query: unbalanced parentheses.");
}
-
- filters.push_back(id_filter);
-
- continue;
+ operatorStack.pop();
+ } else {
+ postfix.push(expression);
}
+ }
- if(search_schema.count(field_name) == 0) {
- return Option<bool>(404, "Could not find a filter field named `" + field_name + "` in the schema.");
+ while (!operatorStack.empty()) {
+ if (operatorStack.top() == "(") {
+ return Option<bool>(400, "Could not parse the filter query: unbalanced parentheses.");
}
+ postfix.push(operatorStack.top());
+ operatorStack.pop();
+ }
- field _field = search_schema.at(field_name);
- std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);
- StringUtils::trim(raw_value);
- filter f;
+ return Option<bool>(true);
+}
- // skip past optional `:=` operator, which has no meaning for non-string fields
- if(!_field.is_string() && raw_value[0] == '=') {
- size_t filter_value_index = 0;
- while(raw_value[++filter_value_index] == ' ');
+Option<bool> toFilter(const std::string expression,
+ filter& filter_exp,
+ const tsl::htrie_map<char, field>& search_schema,
+ const Store* store,
+ const std::string& doc_id_prefix) {
+ // split into [field_name, value]
+ size_t found_index = expression.find(':');
+ if (found_index == std::string::npos) {
+ return Option<bool>(400, "Could not parse the filter query.");
+ }
+ std::string&& field_name = expression.substr(0, found_index);
+ StringUtils::trim(field_name);
+ if (field_name == "id") {
+ std::string&& raw_value = expression.substr(found_index + 1, std::string::npos);
+ StringUtils::trim(raw_value);
+ std::string empty_filter_err = "Error with filter field `id`: Filter value cannot be empty.";
+ if (raw_value.empty()) {
+ return Option<bool>(400, empty_filter_err);
+ }
+ filter_exp = {field_name, {}, {}};
+ NUM_COMPARATOR id_comparator = EQUALS;
+ size_t filter_value_index = 0;
+ if (raw_value[0] == '=') {
+ id_comparator = EQUALS;
+ while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ } else if (raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
+ return Option<bool>(400, "Not equals filtering is not supported on the `id` field.");
+ }
+ if (filter_value_index != 0) {
raw_value = raw_value.substr(filter_value_index);
}
-
- if(_field.is_integer() || _field.is_float()) {
- // could be a single value or a list
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
-
- f = {field_name, {}, {}};
-
- for(std::string & filter_value: filter_values) {
- Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(filter_value);
- if(!op_comparator.ok()) {
- return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());
- }
-
- if(op_comparator.get() == RANGE_INCLUSIVE) {
- // split the value around range operator to extract bounds
- std::vector<std::string> range_values;
- StringUtils::split(filter_value, range_values, filter::RANGE_OPERATOR());
- for(const std::string& range_value: range_values) {
- auto validate_op = filter::validate_numerical_filter_value(_field, range_value);
- if(!validate_op.ok()) {
- return validate_op;
- }
-
- f.values.push_back(range_value);
- f.comparators.push_back(op_comparator.get());
- }
- } else {
- auto validate_op = filter::validate_numerical_filter_value(_field, filter_value);
- if(!validate_op.ok()) {
- return validate_op;
- }
-
- f.values.push_back(filter_value);
- f.comparators.push_back(op_comparator.get());
- }
+ if (raw_value.empty()) {
+ return Option<bool>(400, empty_filter_err);
+ }
+ if (raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> doc_ids;
+ StringUtils::split_to_values(raw_value.substr(1, raw_value.size() - 2), doc_ids);
+ for (std::string& doc_id: doc_ids) {
+ // we have to convert the doc_id to seq id
+ std::string seq_id_str;
+ StoreStatus seq_id_status = store->get(doc_id_prefix + doc_id, seq_id_str);
+ if (seq_id_status != StoreStatus::FOUND) {
+ continue;
}
-
- } else {
- Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);
- if(!op_comparator.ok()) {
+ filter_exp.values.push_back(seq_id_str);
+ filter_exp.comparators.push_back(id_comparator);
+ }
+ } else {
+ std::vector<std::string> doc_ids;
+ StringUtils::split_to_values(raw_value, doc_ids); // to handle backticks
+ std::string seq_id_str;
+ StoreStatus seq_id_status = store->get(doc_id_prefix + doc_ids[0], seq_id_str);
+ if (seq_id_status == StoreStatus::FOUND) {
+ filter_exp.values.push_back(seq_id_str);
+ filter_exp.comparators.push_back(id_comparator);
+ }
+ }
+ return Option<bool>(true);
+ }
+ if (search_schema.count(field_name) == 0) {
+ return Option<bool>(404, "Could not find a filter field named `" + field_name + "` in the schema.");
+ }
+ field _field = search_schema.at(field_name);
+ std::string&& raw_value = expression.substr(found_index + 1, std::string::npos);
+ StringUtils::trim(raw_value);
+ // skip past optional `:=` operator, which has no meaning for non-string fields
+ if (!_field.is_string() && raw_value[0] == '=') {
+ size_t filter_value_index = 0;
+ while (raw_value[++filter_value_index] == ' ');
+ raw_value = raw_value.substr(filter_value_index);
+ }
+ if (_field.is_integer() || _field.is_float()) {
+ // could be a single value or a list
+ if (raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+ filter_exp = {field_name, {}, {}};
+ for (std::string& filter_value: filter_values) {
+ Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(filter_value);
+ if (!op_comparator.ok()) {
return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());
}
-
- if(op_comparator.get() == RANGE_INCLUSIVE) {
+ if (op_comparator.get() == RANGE_INCLUSIVE) {
// split the value around range operator to extract bounds
std::vector<std::string> range_values;
- StringUtils::split(raw_value, range_values, filter::RANGE_OPERATOR());
-
- f.field_name = field_name;
- for(const std::string& range_value: range_values) {
+ StringUtils::split(filter_value, range_values, filter::RANGE_OPERATOR());
+ for (const std::string& range_value: range_values) {
auto validate_op = filter::validate_numerical_filter_value(_field, range_value);
- if(!validate_op.ok()) {
+ if (!validate_op.ok()) {
return validate_op;
}
-
- f.values.push_back(range_value);
- f.comparators.push_back(op_comparator.get());
+ filter_exp.values.push_back(range_value);
+ filter_exp.comparators.push_back(op_comparator.get());
}
} else {
- auto validate_op = filter::validate_numerical_filter_value(_field, raw_value);
- if(!validate_op.ok()) {
+ auto validate_op = filter::validate_numerical_filter_value(_field, filter_value);
+ if (!validate_op.ok()) {
return validate_op;
}
- f = {field_name, {raw_value}, {op_comparator.get()}};
+ filter_exp.values.push_back(filter_value);
+ filter_exp.comparators.push_back(op_comparator.get());
}
}
- } else if(_field.is_bool()) {
- NUM_COMPARATOR bool_comparator = EQUALS;
- size_t filter_value_index = 0;
-
- if(raw_value[0] == '=') {
- bool_comparator = EQUALS;
- while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
- } else if(raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
- bool_comparator = NOT_EQUALS;
- filter_value_index++;
- while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
- }
-
- if(filter_value_index != 0) {
- raw_value = raw_value.substr(filter_value_index);
- }
-
- if(filter_value_index == raw_value.size()) {
- return Option<bool>(400, "Error with filter field `" + _field.name +
- "`: Filter value cannot be empty.");
+ } else {
+ Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);
+ if (!op_comparator.ok()) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());
}
-
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
- f = {field_name, {}, {}};
-
- for(std::string & filter_value: filter_values) {
- if(filter_value != "true" && filter_value != "false") {
- return Option<bool>(400, "Values of filter field `" + _field.name +
- "`: must be `true` or `false`.");
+ if (op_comparator.get() == RANGE_INCLUSIVE) {
+ // split the value around range operator to extract bounds
+ std::vector<std::string> range_values;
+ StringUtils::split(raw_value, range_values, filter::RANGE_OPERATOR());
+ filter_exp.field_name = field_name;
+ for (const std::string& range_value: range_values) {
+ auto validate_op = filter::validate_numerical_filter_value(_field, range_value);
+ if (!validate_op.ok()) {
+ return validate_op;
}
-
- filter_value = (filter_value == "true") ? "1" : "0";
- f.values.push_back(filter_value);
- f.comparators.push_back(bool_comparator);
+ filter_exp.values.push_back(range_value);
+ filter_exp.comparators.push_back(op_comparator.get());
}
} else {
- if(raw_value != "true" && raw_value != "false") {
- return Option<bool>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");
+ auto validate_op = filter::validate_numerical_filter_value(_field, raw_value);
+ if (!validate_op.ok()) {
+ return validate_op;
}
-
- std::string bool_value = (raw_value == "true") ? "1" : "0";
- f = {field_name, {bool_value}, {bool_comparator}};
+ filter_exp = {field_name, {raw_value}, {op_comparator.get()}};
}
-
- } else if(_field.is_geopoint()) {
- f = {field_name, {}, {}};
-
- const std::string& format_err_msg = "Value of filter field `" + _field.name +
- "`: must be in the `(-44.50, 170.29, 0.75 km)` or "
- "(56.33, -65.97, 23.82, -127.82) format.";
-
- NUM_COMPARATOR num_comparator;
-
- // could be a single value or a list
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, "),");
-
- for(std::string& filter_value: filter_values) {
- filter_value += ")";
- std::string processed_filter_val;
- auto parse_op = parse_geopoint_filter_value(filter_value, format_err_msg, processed_filter_val, num_comparator);
-
- if(!parse_op.ok()) {
- return parse_op;
- }
-
- f.values.push_back(processed_filter_val);
- f.comparators.push_back(num_comparator);
+ }
+ } else if (_field.is_bool()) {
+ NUM_COMPARATOR bool_comparator = EQUALS;
+ size_t filter_value_index = 0;
+ if (raw_value[0] == '=') {
+ bool_comparator = EQUALS;
+ while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ } else if (raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
+ bool_comparator = NOT_EQUALS;
+ filter_value_index++;
+ while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ }
+ if (filter_value_index != 0) {
+ raw_value = raw_value.substr(filter_value_index);
+ }
+ if (filter_value_index == raw_value.size()) {
+ return Option<bool>(400, "Error with filter field `" + _field.name +
+ "`: Filter value cannot be empty.");
+ }
+ if (raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+ filter_exp = {field_name, {}, {}};
+ for (std::string& filter_value: filter_values) {
+ if (filter_value != "true" && filter_value != "false") {
+ return Option<bool>(400, "Values of filter field `" + _field.name +
+ "`: must be `true` or `false`.");
}
- } else {
- // single value, e.g. (10.45, 34.56, 2 km)
+ filter_value = (filter_value == "true") ? "1" : "0";
+ filter_exp.values.push_back(filter_value);
+ filter_exp.comparators.push_back(bool_comparator);
+ }
+ } else {
+ if (raw_value != "true" && raw_value != "false") {
+ return Option<bool>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");
+ }
+ std::string bool_value = (raw_value == "true") ? "1" : "0";
+ filter_exp = {field_name, {bool_value}, {bool_comparator}};
+ }
+ } else if (_field.is_geopoint()) {
+ filter_exp = {field_name, {}, {}};
+ const std::string& format_err_msg = "Value of filter field `" + _field.name +
+ "`: must be in the `(-44.50, 170.29, 0.75 km)` or "
+ "(56.33, -65.97, 23.82, -127.82) format.";
+ NUM_COMPARATOR num_comparator;
+ // could be a single value or a list
+ if (raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, "),");
+ for (std::string& filter_value: filter_values) {
+ filter_value += ")";
std::string processed_filter_val;
- auto parse_op = parse_geopoint_filter_value(raw_value, format_err_msg, processed_filter_val, num_comparator);
-
- if(!parse_op.ok()) {
+ auto parse_op = filter::parse_geopoint_filter_value(filter_value, format_err_msg, processed_filter_val,
+ num_comparator);
+ if (!parse_op.ok()) {
return parse_op;
}
-
- f.values.push_back(processed_filter_val);
- f.comparators.push_back(num_comparator);
+ filter_exp.values.push_back(processed_filter_val);
+ filter_exp.comparators.push_back(num_comparator);
}
-
- } else if(_field.is_string()) {
- size_t filter_value_index = 0;
- NUM_COMPARATOR str_comparator = CONTAINS;
-
- if(raw_value[0] == '=') {
- // string filter should be evaluated in strict "equals" mode
- str_comparator = EQUALS;
- while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
- } else if(raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
- str_comparator = NOT_EQUALS;
- filter_value_index++;
- while(++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ } else {
+ // single value, e.g. (10.45, 34.56, 2 km)
+ std::string processed_filter_val;
+ auto parse_op = filter::parse_geopoint_filter_value(raw_value, format_err_msg, processed_filter_val,
+ num_comparator);
+ if (!parse_op.ok()) {
+ return parse_op;
}
+ filter_exp.values.push_back(processed_filter_val);
+ filter_exp.comparators.push_back(num_comparator);
+ }
+ } else if (_field.is_string()) {
+ size_t filter_value_index = 0;
+ NUM_COMPARATOR str_comparator = CONTAINS;
+ if (raw_value[0] == '=') {
+ // string filter should be evaluated in strict "equals" mode
+ str_comparator = EQUALS;
+ while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ } else if (raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
+ str_comparator = NOT_EQUALS;
+ filter_value_index++;
+ while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ }
+ if (filter_value_index == raw_value.size()) {
+ return Option<bool>(400, "Error with filter field `" + _field.name +
+ "`: Filter value cannot be empty.");
+ }
+ if (raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split_to_values(
+ raw_value.substr(filter_value_index + 1, raw_value.size() - filter_value_index - 2), filter_values);
+ filter_exp = {field_name, filter_values, {str_comparator}};
+ } else {
+ filter_exp = {field_name, {raw_value.substr(filter_value_index)}, {str_comparator}};
+ }
+ } else {
+ return Option<bool>(400, "Error with filter field `" + _field.name +
+ "`: Unidentified field data type, see docs for supported data types.");
+ }
+
+ return Option<bool>(true);
+}
+
+// https://stackoverflow.com/a/423914/11218270
+Option<bool> toParseTree(std::queue<std::string>& postfix, filter_node_t*& root,
+ const tsl::htrie_map<char, field>& search_schema,
+ const Store* store,
+ const std::string& doc_id_prefix) {
+ std::stack<filter_node_t*> nodeStack;
+
+ while (!postfix.empty()) {
+ const std::string expression = postfix.front();
+ postfix.pop();
- if(filter_value_index == raw_value.size()) {
- return Option<bool>(400, "Error with filter field `" + _field.name +
- "`: Filter value cannot be empty.");
+ filter_node_t* filter_node;
+ if (isOperator(expression)) {
+ auto message = "Could not parse the filter query: unbalanced `" + expression + "` operands.";
+
+ if (nodeStack.empty()) {
+ return Option<bool>(400, message);
}
+ auto operandB = nodeStack.top();
+ nodeStack.pop();
- if(raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split_to_values(raw_value.substr(filter_value_index+1, raw_value.size() - filter_value_index - 2), filter_values);
- f = {field_name, filter_values, {str_comparator}};
- } else {
- f = {field_name, {raw_value.substr(filter_value_index)}, {str_comparator}};
+ if (nodeStack.empty()) {
+ return Option<bool>(400, message);
}
- } else {
- return Option<bool>(400, "Error with filter field `" + _field.name +
- "`: Unidentified field data type, see docs for supported data types.");
- }
+ auto operandA = nodeStack.top();
+ nodeStack.pop();
- if(!f.comparators.empty() && f.comparators.front() == NOT_EQUALS) {
- exclude_filters.push_back(f);
+ filter_node = new filter_node_t(expression == "&&" ? AND : OR, operandA, operandB);
} else {
- filters.push_back(f);
+ filter filter_exp;
+ Option<bool> toFilter_op = toFilter(expression, filter_exp, search_schema, store, doc_id_prefix);
+ if (!toFilter_op.ok()) {
+ return toFilter_op;
+ }
+
+ filter_node = new filter_node_t(filter_exp);
}
+
+ nodeStack.push(filter_node);
}
- filters.insert( filters.end(), exclude_filters.begin(), exclude_filters.end() );
+ if (nodeStack.empty()) {
+ return Option<bool>(400, "Filter query cannot be empty.");
+ }
+
+ root = nodeStack.top();
+ return Option<bool>(true);
+}
+
+Option<bool> filter::parse_filter_query(const std::string& filter_query,
+ const tsl::htrie_map<char, field>& search_schema,
+ const Store* store,
+ const std::string& doc_id_prefix,
+ filter_node_t*& root) {
+ auto _filter_query = filter_query;
+ StringUtils::trim(_filter_query);
+ if (_filter_query.empty()) {
+ return Option<bool>(true);
+ }
+
+ std::queue<std::string> tokens;
+ Option<bool> tokenize_op = StringUtils::tokenize(filter_query, tokens);
+ if (!tokenize_op.ok()) {
+ return tokenize_op;
+ }
+
+ if (tokens.size() > 100) {
+ return Option<bool>(400, "Filter expression is not valid.");
+ }
+
+ std::queue<std::string> postfix;
+ Option<bool> toPostfix_op = toPostfix(tokens, postfix);
+ if (!toPostfix_op.ok()) {
+ return toPostfix_op;
+ }
+
+ Option<bool> toParseTree_op = toParseTree(postfix, root, search_schema, store, doc_id_prefix);
+ if (!toParseTree_op.ok()) {
+ return toParseTree_op;
+ }
return Option<bool>(true);
}
@@ -390,7 +458,7 @@ Option<bool> field::json_field_to_field(nlohmann::json& field_json, std::vector<
if(field_json.count("drop") != 0) {
return Option<bool>(400, std::string("Invalid property `drop` on field `") +
field_json[fields::name].get<std::string>() + std::string("`: it is allowed only "
- "during schema update."));
+ "during schema update."));
}
if(field_json.count(fields::facet) != 0 && !field_json.at(fields::facet).is_boolean()) {
@@ -562,10 +630,10 @@ Option<bool> field::json_field_to_field(nlohmann::json& field_json, std::vector<
auto vec_dist = magic_enum::enum_cast<vector_distance_type_t>(field_json[fields::vec_dist].get<std::string>()).value();
the_fields.emplace_back(
- field(field_json[fields::name], field_json[fields::type], field_json[fields::facet],
- field_json[fields::optional], field_json[fields::index], field_json[fields::locale],
- field_json[fields::sort], field_json[fields::infix], field_json[fields::nested],
- field_json[fields::nested_array], field_json[fields::num_dim], vec_dist)
+ field(field_json[fields::name], field_json[fields::type], field_json[fields::facet],
+ field_json[fields::optional], field_json[fields::index], field_json[fields::locale],
+ field_json[fields::sort], field_json[fields::infix], field_json[fields::nested],
+ field_json[fields::nested_array], field_json[fields::num_dim], vec_dist)
);
return Option<bool>(true);
@@ -613,8 +681,8 @@ bool field::flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_arr
}
Option<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, const field& the_field,
- std::vector<std::string>& path_parts, size_t path_index,
- bool has_array, bool has_obj_array, std::unordered_map<std::string, field>& flattened_fields) {
+ std::vector<std::string>& path_parts, size_t path_index,
+ bool has_array, bool has_obj_array, std::unordered_map<std::string, field>& flattened_fields) {
if(path_index == path_parts.size()) {
// end of path: check if obj matches expected type
std::string detected_type;
@@ -631,10 +699,10 @@ Option<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, cons
// handle differences in detection of numerical types
bool is_numericaly_valid = (detected_type != the_field.type) &&
- ((detected_type == field_types::INT64 &&
- (the_field.type == field_types::INT32 || the_field.type == field_types::FLOAT)) ||
- (detected_type == field_types::INT64_ARRAY &&
- (the_field.type == field_types::INT32_ARRAY || the_field.type == field_types::FLOAT_ARRAY)));
+ ((detected_type == field_types::INT64 &&
+ (the_field.type == field_types::INT32 || the_field.type == field_types::FLOAT)) ||
+ (detected_type == field_types::INT64_ARRAY &&
+ (the_field.type == field_types::INT32_ARRAY || the_field.type == field_types::FLOAT_ARRAY)));
if(detected_type == the_field.type || is_numericaly_valid) {
if(the_field.is_object()) {
@@ -660,7 +728,7 @@ Option<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, cons
} else {
if(has_obj_array && !the_field.is_array()) {
return Option<bool>(400, "Field `" + the_field.name + "` has an incorrect type. "
- "Hint: field inside an array of objects must be an array type as well.");
+ "Hint: field inside an array of objects must be an array type as well.");
}
return Option<bool>(400, "Field `" + the_field.name + "` has an incorrect type.");
@@ -754,4 +822,3 @@ Option<bool> field::flatten_stored_doc(nlohmann::json& document, const tsl::htri
return Option<bool>(true);
}
-
diff --git a/src/index.cpp b/src/index.cpp
index 57fde5c5..924b2e77 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -46,7 +46,7 @@ struct token_posting_t {
const posting_list_t::iterator_t& posting;
token_posting_t(uint32_t token_id, const posting_list_t::iterator_t& posting):
- token_id(token_id), posting(posting) {
+ token_id(token_id), posting(posting) {
}
};
@@ -243,9 +243,9 @@ float Index::int64_t_to_float(int64_t n) {
}
void Index::compute_token_offsets_facets(index_record& record,
- const tsl::htrie_map<char, field>& search_schema,
- const std::vector<char>& local_token_separators,
- const std::vector<char>& local_symbols_to_index) {
+ const tsl::htrie_map<char, field>& search_schema,
+ const std::vector<char>& local_token_separators,
+ const std::vector<char>& local_symbols_to_index) {
const auto& document = record.doc;
@@ -410,7 +410,7 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3
if((op != UPDATE && op != EMPLACE) && missing_default_sort_field) {
return Option<>(400, "Field `" + default_sorting_field + "` has been declared as a default sorting field, "
- "but is not found in the document.");
+ "but is not found in the document.");
}
for(const auto& a_field: search_schema) {
@@ -426,7 +426,7 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3
if(document.count(field_name) == 0) {
return Option<>(400, "Field `" + field_name + "` has been declared in the schema, "
- "but is not found in the document.");
+ "but is not found in the document.");
}
if(a_field.optional && document[field_name].is_null()) {
@@ -484,7 +484,7 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3
} else if(a_field.is_array()) {
if(!document[field_name].is_array()) {
if(a_field.optional && (dirty_values == DIRTY_VALUES::DROP ||
- dirty_values == DIRTY_VALUES::COERCE_OR_DROP)) {
+ dirty_values == DIRTY_VALUES::COERCE_OR_DROP)) {
document.erase(field_name);
continue;
} else {
@@ -549,13 +549,13 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3
}
void Index::validate_and_preprocess(Index *index, std::vector<index_record>& iter_batch,
- const size_t batch_start_index, const size_t batch_size,
- const std::string& default_sorting_field,
- const tsl::htrie_map<char, field>& search_schema,
- const std::string& fallback_field_type,
- const std::vector<char>& token_separators,
- const std::vector<char>& symbols_to_index,
- const bool do_validation) {
+ const size_t batch_start_index, const size_t batch_size,
+ const std::string& default_sorting_field,
+ const tsl::htrie_map<char, field>& search_schema,
+ const std::string& fallback_field_type,
+ const std::vector<char>& token_separators,
+ const std::vector<char>& symbols_to_index,
+ const bool do_validation) {
// runs in a partitioned thread
@@ -847,10 +847,10 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
else if(afield.type == field_types::INT64) {
auto num_tree = numerical_index.at(afield.name);
iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree]
- (const index_record& record, uint32_t seq_id) {
+ (const index_record& record, uint32_t seq_id) {
int64_t value = record.doc[afield.name].get<int64_t>();
num_tree->insert(value, seq_id);
- });
+ });
}
else if(afield.type == field_types::FLOAT) {
@@ -888,34 +888,34 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
auto geo_index = geopoint_index.at(afield.name);
iterate_and_index_numerical_field(iter_batch, afield,
- [&afield, &geo_array_index=geo_array_index, geo_index](const index_record& record, uint32_t seq_id) {
-
- const std::vector<std::vector<double>>& latlongs = record.doc[afield.name];
- S2RegionTermIndexer::Options options;
- options.set_index_contains_points_only(true);
- S2RegionTermIndexer indexer(options);
-
- int64_t* packed_latlongs = new int64_t[latlongs.size() + 1];
- packed_latlongs[0] = latlongs.size();
-
- for(size_t li = 0; li < latlongs.size(); li++) {
- auto& latlong = latlongs[li];
- S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();
- std::set<std::string> terms;
- for(const auto& term: indexer.GetIndexTerms(point, "")) {
- terms.insert(term);
- }
-
- for(const auto& term: terms) {
- (*geo_index)[term].push_back(seq_id);
- }
-
- int64_t packed_latlong = GeoPoint::pack_lat_lng(latlong[0], latlong[1]);
- packed_latlongs[li + 1] = packed_latlong;
- }
-
- geo_array_index.at(afield.name)->emplace(seq_id, packed_latlongs);
- });
+ [&afield, &geo_array_index=geo_array_index, geo_index](const index_record& record, uint32_t seq_id) {
+
+ const std::vector<std::vector<double>>& latlongs = record.doc[afield.name];
+ S2RegionTermIndexer::Options options;
+ options.set_index_contains_points_only(true);
+ S2RegionTermIndexer indexer(options);
+
+ int64_t* packed_latlongs = new int64_t[latlongs.size() + 1];
+ packed_latlongs[0] = latlongs.size();
+
+ for(size_t li = 0; li < latlongs.size(); li++) {
+ auto& latlong = latlongs[li];
+ S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();
+ std::set<std::string> terms;
+ for(const auto& term: indexer.GetIndexTerms(point, "")) {
+ terms.insert(term);
+ }
+
+ for(const auto& term: terms) {
+ (*geo_index)[term].push_back(seq_id);
+ }
+
+ int64_t packed_latlong = GeoPoint::pack_lat_lng(latlong[0], latlong[1]);
+ packed_latlongs[li + 1] = packed_latlong;
+ }
+
+ geo_array_index.at(afield.name)->emplace(seq_id, packed_latlongs);
+ });
} else if(afield.is_array()) {
// all other numerical arrays
auto num_tree = numerical_index.at(afield.name);
@@ -1025,20 +1025,6 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
}
}
-void Index::insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
- const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const {
- for(auto & kv: token_to_offsets) {
- art_document art_doc(seq_id, score, kv.second);
- uint32_t num_hits = 0;
-
- const unsigned char *key = (const unsigned char *) kv.first.c_str();
- int key_len = (int) kv.first.length() + 1; // for the terminating \0 char
-
- //LOG(INFO) << "key: " << key << ", art_doc.id: " << art_doc.id;
- art_insert(t, key, key_len, &art_doc);
- }
-}
-
uint64_t Index::facet_token_hash(const field & a_field, const std::string &token) {
// for integer/float use their native values
uint64_t hash = 0;
@@ -1304,7 +1290,7 @@ void Index::search_all_candidates(const size_t num_search_fields,
// escape hatch to prevent too much looping but subject to being overriden explicitly via `max_candidates`
long long combination_limit = (num_search_fields == 1 && prefixes[0]) ? max_candidates :
- std::max<size_t>(Index::COMBINATION_MIN_LIMIT, max_candidates);
+ std::max<size_t>(Index::COMBINATION_MIN_LIMIT, max_candidates);
for(long long n = 0; n < N && n < combination_limit; ++n) {
RETURN_CIRCUIT_BREAKER
@@ -1411,7 +1397,7 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
// Prepare excluded document IDs that we can later remove from the result set
uint32_t* excluded_result_ids = nullptr;
size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,
- &curated_ids[0], curated_ids.size(), &excluded_result_ids);
+ &curated_ids[0], curated_ids.size(), &excluded_result_ids);
std::vector<void*> posting_lists;
@@ -1420,7 +1406,7 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
}
result_iter_state_t iter_state(
- excluded_result_ids, excluded_result_ids_size, filter_ids, filter_ids_length
+ excluded_result_ids, excluded_result_ids_size, filter_ids, filter_ids_length
);
// We fetch offset positions only for multi token query
@@ -1438,28 +1424,28 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
if(topster == nullptr) {
posting_t::block_intersector_t(
- posting_lists, iter_state, thread_pool, 100
+ posting_lists, iter_state, thread_pool, 100
)
- .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its, size_t index) {
- result_id_vecs[index].push_back(seq_id);
- }, concurrency);
+ .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its, size_t index) {
+ result_id_vecs[index].push_back(seq_id);
+ }, concurrency);
} else {
for(size_t i = 0; i < concurrency; i++) {
topsters[i] = new Topster(topster->MAX_SIZE, topster->distinct);
}
posting_t::block_intersector_t(
- posting_lists, iter_state, thread_pool, 100
+ posting_lists, iter_state, thread_pool, 100
)
- .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its, size_t index) {
- score_results(sort_fields, searched_queries.size(), field_id, field_is_array,
- total_cost, topsters[index], query_suggestion, groups_processed_vec[index],
- seq_id, sort_order, field_values, geopoint_indices,
- group_limit, group_by_fields, token_bits,
- prioritize_exact_match, single_exact_query_token, syn_orig_num_tokens, its);
+ .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its, size_t index) {
+ score_results(sort_fields, searched_queries.size(), field_id, field_is_array,
+ total_cost, topsters[index], query_suggestion, groups_processed_vec[index],
+ seq_id, sort_order, field_values, geopoint_indices,
+ group_limit, group_by_fields, token_bits,
+ prioritize_exact_match, single_exact_query_token, syn_orig_num_tokens, its);
- result_id_vecs[index].push_back(seq_id);
- }, concurrency);
+ result_id_vecs[index].push_back(seq_id);
+ }, concurrency);
}
delete [] excluded_result_ids;
@@ -1515,322 +1501,305 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
}
}
-void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,
- const std::vector<filter>& filters,
- const bool enable_short_circuit) const {
- //auto begin = std::chrono::high_resolution_clock::now();
- for(size_t i = 0; i < filters.size(); i++) {
- const filter & a_filter = filters[i];
-
- if(a_filter.field_name == "id") {
- // we handle `ids` separately
- std::vector<uint32> result_ids;
- for(const auto& id_str: a_filter.values) {
- result_ids.push_back(std::stoul(id_str));
- }
-
- std::sort(result_ids.begin(), result_ids.end());
-
- if(i == 0) {
- filter_ids = new uint32[result_ids.size()];
- std::copy(result_ids.begin(), result_ids.end(), filter_ids);
- filter_ids_length = result_ids.size();
- } else {
- uint32_t* filtered_results = nullptr;
- filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, &result_ids[0],
- result_ids.size(), &filtered_results);
- delete [] filter_ids;
- filter_ids = filtered_results;
- }
+void Index::do_filtering(uint32_t*& filter_ids,
+ uint32_t& filter_ids_length,
+ filter_node_t const* const root) const {
+ // auto begin = std::chrono::high_resolution_clock::now();
+ const filter a_filter = root->filter_exp;
- continue;
+ if (a_filter.field_name == "id") {
+ // we handle `ids` separately
+ std::vector<uint32> result_ids;
+ for (const auto& id_str : a_filter.values) {
+ result_ids.push_back(std::stoul(id_str));
}
- bool has_search_index = search_index.count(a_filter.field_name) != 0 ||
- numerical_index.count(a_filter.field_name) != 0 ||
- geopoint_index.count(a_filter.field_name) != 0;
+ std::sort(result_ids.begin(), result_ids.end());
- if(!has_search_index) {
- continue;
+ if (filter_ids_length == 0) {
+ filter_ids = new uint32[result_ids.size()];
+ std::copy(result_ids.begin(), result_ids.end(), filter_ids);
+ filter_ids_length = result_ids.size();
+ } else {
+ uint32_t* filtered_results = nullptr;
+ filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, &result_ids[0],
+ result_ids.size(), &filtered_results);
+ delete[] filter_ids;
+ filter_ids = filtered_results;
}
- field f = search_schema.at(a_filter.field_name);
+ return;
+ }
- uint32_t* result_ids = nullptr;
- size_t result_ids_len = 0;
+ bool has_search_index = search_index.count(a_filter.field_name) != 0 ||
+ numerical_index.count(a_filter.field_name) != 0 ||
+ geopoint_index.count(a_filter.field_name) != 0;
- if(f.is_integer()) {
- auto num_tree = numerical_index.at(a_filter.field_name);
+ if (!has_search_index) {
+ return;
+ }
- for(size_t fi=0; fi < a_filter.values.size(); fi++) {
- const std::string & filter_value = a_filter.values[fi];
- int64_t value = (int64_t) std::stol(filter_value);
+ field f = search_schema.at(a_filter.field_name);
- if(a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {
- const std::string& next_filter_value = a_filter.values[fi+1];
- int64_t range_end_value = (int64_t) std::stol(next_filter_value);
- num_tree->range_inclusive_search(value, range_end_value, &result_ids, result_ids_len);
- fi++;
- } else {
- num_tree->search(a_filter.comparators[fi], value, &result_ids, result_ids_len);
- }
- }
+ uint32_t* result_ids = nullptr;
+ size_t result_ids_len = 0;
- } else if(f.is_float()) {
- auto num_tree = numerical_index.at(a_filter.field_name);
+ if (f.is_integer()) {
+ auto num_tree = numerical_index.at(a_filter.field_name);
- for(size_t fi=0; fi < a_filter.values.size(); fi++) {
- const std::string & filter_value = a_filter.values[fi];
- float value = (float) std::atof(filter_value.c_str());
- int64_t float_int64 = float_to_int64_t(value);
+ for (size_t fi = 0; fi < a_filter.values.size(); fi++) {
+ const std::string& filter_value = a_filter.values[fi];
+ int64_t value = (int64_t)std::stol(filter_value);
- if(a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {
- const std::string& next_filter_value = a_filter.values[fi+1];
- int64_t range_end_value = float_to_int64_t((float) std::atof(next_filter_value.c_str()));
- num_tree->range_inclusive_search(float_int64, range_end_value, &result_ids, result_ids_len);
- fi++;
- } else {
- num_tree->search(a_filter.comparators[fi], float_int64, &result_ids, result_ids_len);
- }
+ if(a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {
+ const std::string& next_filter_value = a_filter.values[fi + 1];
+ int64_t range_end_value = (int64_t)std::stol(next_filter_value);
+ num_tree->range_inclusive_search(value, range_end_value, &result_ids, result_ids_len);
+ fi++;
+ } else {
+ num_tree->search(a_filter.comparators[fi], value, &result_ids, result_ids_len);
}
+ }
+ } else if (f.is_float()) {
+ auto num_tree = numerical_index.at(a_filter.field_name);
- } else if(f.is_bool()) {
- auto num_tree = numerical_index.at(a_filter.field_name);
+ for (size_t fi = 0; fi < a_filter.values.size(); fi++) {
+ const std::string& filter_value = a_filter.values[fi];
+ float value = (float)std::atof(filter_value.c_str());
+ int64_t float_int64 = float_to_int64_t(value);
- size_t value_index = 0;
- for(const std::string & filter_value: a_filter.values) {
- int64_t bool_int64 = (filter_value == "1") ? 1 : 0;
- if(a_filter.comparators[value_index] == NOT_EQUALS) {
- uint32_t* to_exclude_ids = nullptr;
- size_t to_exclude_ids_len = 0;
- num_tree->search(EQUALS, bool_int64, &to_exclude_ids, to_exclude_ids_len);
+ if(a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {
+ const std::string& next_filter_value = a_filter.values[fi+1];
+ int64_t range_end_value = float_to_int64_t((float) std::atof(next_filter_value.c_str()));
+ num_tree->range_inclusive_search(float_int64, range_end_value, &result_ids, result_ids_len);
+ fi++;
+ } else {
+ num_tree->search(a_filter.comparators[fi], float_int64, &result_ids, result_ids_len);
+ }
+ }
+ } else if (f.is_bool()) {
+ auto num_tree = numerical_index.at(a_filter.field_name);
- auto all_ids = seq_ids->uncompress();
- auto all_ids_size = seq_ids->num_ids();
+ size_t value_index = 0;
+ for (const std::string& filter_value : a_filter.values) {
+ int64_t bool_int64 = (filter_value == "1") ? 1 : 0;
+ if (a_filter.comparators[value_index] == NOT_EQUALS) {
+ uint32_t* to_exclude_ids = nullptr;
+ size_t to_exclude_ids_len = 0;
+ num_tree->search(EQUALS, bool_int64, &to_exclude_ids, to_exclude_ids_len);
- uint32_t* excluded_ids = nullptr;
- size_t excluded_ids_len = 0;
+ auto all_ids = seq_ids->uncompress();
+ auto all_ids_size = seq_ids->num_ids();
- excluded_ids_len = ArrayUtils::exclude_scalar(all_ids, all_ids_size, to_exclude_ids,
- to_exclude_ids_len, &excluded_ids);
+ uint32_t* excluded_ids = nullptr;
+ size_t excluded_ids_len = 0;
- delete [] all_ids;
- delete [] to_exclude_ids;
+ excluded_ids_len = ArrayUtils::exclude_scalar(all_ids, all_ids_size, to_exclude_ids,
+ to_exclude_ids_len, &excluded_ids);
- uint32_t *out = nullptr;
- result_ids_len = ArrayUtils::or_scalar(result_ids, result_ids_len,
- excluded_ids, excluded_ids_len, &out);
- delete [] result_ids;
- result_ids = out;
- delete [] excluded_ids;
- } else {
- num_tree->search(a_filter.comparators[value_index], bool_int64, &result_ids, result_ids_len);
- }
+ delete[] all_ids;
+ delete[] to_exclude_ids;
- value_index++;
+ uint32_t* out = nullptr;
+ result_ids_len = ArrayUtils::or_scalar(result_ids, result_ids_len,
+ excluded_ids, excluded_ids_len, &out);
+ delete[] result_ids;
+ result_ids = out;
+ delete[] excluded_ids;
+ } else {
+ num_tree->search(a_filter.comparators[value_index], bool_int64, &result_ids, result_ids_len);
}
- } else if(f.is_geopoint()) {
- for(const std::string& filter_value: a_filter.values) {
- std::vector<uint32_t> geo_result_ids;
+ value_index++;
+ }
+ } else if (f.is_geopoint()) {
+ for (const std::string& filter_value : a_filter.values) {
+ std::vector<uint32_t> geo_result_ids;
- std::vector<std::string> filter_value_parts;
- StringUtils::split(filter_value, filter_value_parts, ","); // x, y, 2, km (or) list of points
+ std::vector<std::string> filter_value_parts;
+ StringUtils::split(filter_value, filter_value_parts, ","); // x, y, 2, km (or) list of points
- bool is_polygon = StringUtils::is_float(filter_value_parts.back());
- S2Region* query_region;
+ bool is_polygon = StringUtils::is_float(filter_value_parts.back());
+ S2Region* query_region;
- if(is_polygon) {
- const int num_verts = int(filter_value_parts.size()) / 2;
- std::vector<S2Point> vertices;
- double sum = 0.0;
+ if (is_polygon) {
+ const int num_verts = int(filter_value_parts.size()) / 2;
+ std::vector<S2Point> vertices;
+ double sum = 0.0;
- for(size_t point_index = 0; point_index < size_t(num_verts); point_index++) {
- double lat = std::stod(filter_value_parts[point_index * 2]);
- double lon = std::stod(filter_value_parts[point_index * 2 + 1]);
- S2Point vertex = S2LatLng::FromDegrees(lat, lon).ToPoint();
- vertices.emplace_back(vertex);
- }
+ for (size_t point_index = 0; point_index < size_t(num_verts);
+ point_index++) {
+ double lat = std::stod(filter_value_parts[point_index * 2]);
+ double lon = std::stod(filter_value_parts[point_index * 2 + 1]);
+ S2Point vertex = S2LatLng::FromDegrees(lat, lon).ToPoint();
+ vertices.emplace_back(vertex);
+ }
- auto loop = new S2Loop(vertices, S2Debug::DISABLE);
- loop->Normalize(); // if loop is not CCW but CW, change to CCW.
+ auto loop = new S2Loop(vertices, S2Debug::DISABLE);
+ loop->Normalize(); // if loop is not CCW but CW, change to CCW.
- S2Error error;
- if (loop->FindValidationError(&error)) {
- LOG(ERROR) << "Query vertex is bad, skipping. Error: " << error;
- delete loop;
- continue;
- } else {
- query_region = loop;
- }
+ S2Error error;
+ if (loop->FindValidationError(&error)) {
+ LOG(ERROR) << "Query vertex is bad, skipping. Error: " << error;
+ delete loop;
+ continue;
} else {
- double radius = std::stof(filter_value_parts[2]);
- const auto& unit = filter_value_parts[3];
-
- if(unit == "km") {
- radius *= 1000;
- } else {
- // assume "mi" (validated upstream)
- radius *= 1609.34;
- }
+ query_region = loop;
+ }
+ } else {
+ double radius = std::stof(filter_value_parts[2]);
+ const auto& unit = filter_value_parts[3];
- S1Angle query_radius = S1Angle::Radians(S2Earth::MetersToRadians(radius));
- double query_lat = std::stod(filter_value_parts[0]);
- double query_lng = std::stod(filter_value_parts[1]);
- S2Point center = S2LatLng::FromDegrees(query_lat, query_lng).ToPoint();
- query_region = new S2Cap(center, query_radius);
+ if (unit == "km") {
+ radius *= 1000;
+ } else {
+ // assume "mi" (validated upstream)
+ radius *= 1609.34;
}
- S2RegionTermIndexer::Options options;
- options.set_index_contains_points_only(true);
- S2RegionTermIndexer indexer(options);
+ S1Angle query_radius = S1Angle::Radians(S2Earth::MetersToRadians(radius));
+ double query_lat = std::stod(filter_value_parts[0]);
+ double query_lng = std::stod(filter_value_parts[1]);
+ S2Point center = S2LatLng::FromDegrees(query_lat, query_lng).ToPoint();
+ query_region = new S2Cap(center, query_radius);
+ }
- for (const auto& term : indexer.GetQueryTerms(*query_region, "")) {
- auto geo_index = geopoint_index.at(a_filter.field_name);
- const auto& ids_it = geo_index->find(term);
- if(ids_it != geo_index->end()) {
- geo_result_ids.insert(geo_result_ids.end(), ids_it->second.begin(), ids_it->second.end());
- }
+ S2RegionTermIndexer::Options options;
+ options.set_index_contains_points_only(true);
+ S2RegionTermIndexer indexer(options);
+
+ for (const auto& term : indexer.GetQueryTerms(*query_region, "")) {
+ auto geo_index = geopoint_index.at(a_filter.field_name);
+ const auto& ids_it = geo_index->find(term);
+ if(ids_it != geo_index->end()) {
+ geo_result_ids.insert(geo_result_ids.end(), ids_it->second.begin(), ids_it->second.end());
}
+ }
- gfx::timsort(geo_result_ids.begin(), geo_result_ids.end());
- geo_result_ids.erase(std::unique( geo_result_ids.begin(), geo_result_ids.end() ), geo_result_ids.end());
+ gfx::timsort(geo_result_ids.begin(), geo_result_ids.end());
+ geo_result_ids.erase(std::unique( geo_result_ids.begin(), geo_result_ids.end() ), geo_result_ids.end());
- // `geo_result_ids` will contain all IDs that are within approximately within query radius
- // we still need to do another round of exact filtering on them
+ // `geo_result_ids` will contain all IDs that are within approximately within query radius
+ // we still need to do another round of exact filtering on them
- std::vector<uint32_t> exact_geo_result_ids;
+ std::vector<uint32_t> exact_geo_result_ids;
- if(f.is_single_geopoint()) {
- spp::sparse_hash_map<uint32_t, int64_t>* sort_field_index = sort_index.at(f.name);
+ if (f.is_single_geopoint()) {
+ spp::sparse_hash_map<uint32_t, int64_t>* sort_field_index = sort_index.at(f.name);
- for(auto result_id: geo_result_ids) {
- // no need to check for existence of `result_id` because of indexer based pre-filtering above
- int64_t lat_lng = sort_field_index->at(result_id);
- S2LatLng s2_lat_lng;
- GeoPoint::unpack_lat_lng(lat_lng, s2_lat_lng);
- if (query_region->Contains(s2_lat_lng.ToPoint())) {
- exact_geo_result_ids.push_back(result_id);
- }
+ for (auto result_id : geo_result_ids) {
+ // no need to check for existence of `result_id` because of indexer based pre-filtering above
+ int64_t lat_lng = sort_field_index->at(result_id);
+ S2LatLng s2_lat_lng;
+ GeoPoint::unpack_lat_lng(lat_lng, s2_lat_lng);
+ if (query_region->Contains(s2_lat_lng.ToPoint())) {
+ exact_geo_result_ids.push_back(result_id);
}
- } else {
- spp::sparse_hash_map<uint32_t, int64_t*>* geo_field_index = geo_array_index.at(f.name);
+ }
+ } else {
+ spp::sparse_hash_map<uint32_t, int64_t*>* geo_field_index = geo_array_index.at(f.name);
- for(auto result_id: geo_result_ids) {
- int64_t* lat_lngs = geo_field_index->at(result_id);
+ for (auto result_id : geo_result_ids) {
+ int64_t* lat_lngs = geo_field_index->at(result_id);
- bool point_found = false;
+ bool point_found = false;
- // any one point should exist
- for(size_t li = 0; li < lat_lngs[0]; li++) {
- int64_t lat_lng = lat_lngs[li + 1];
- S2LatLng s2_lat_lng;
- GeoPoint::unpack_lat_lng(lat_lng, s2_lat_lng);
- if (query_region->Contains(s2_lat_lng.ToPoint())) {
- point_found = true;
- break;
- }
+ // any one point should exist
+ for (size_t li = 0; li < lat_lngs[0]; li++) {
+ int64_t lat_lng = lat_lngs[li + 1];
+ S2LatLng s2_lat_lng;
+ GeoPoint::unpack_lat_lng(lat_lng, s2_lat_lng);
+ if (query_region->Contains(s2_lat_lng.ToPoint())) {
+ point_found = true;
+ break;
}
+ }
- if(point_found) {
- exact_geo_result_ids.push_back(result_id);
- }
+ if (point_found) {
+ exact_geo_result_ids.push_back(result_id);
}
}
-
- uint32_t *out = nullptr;
- result_ids_len = ArrayUtils::or_scalar(&exact_geo_result_ids[0], exact_geo_result_ids.size(),
- result_ids, result_ids_len, &out);
-
- delete [] result_ids;
- result_ids = out;
-
- delete query_region;
}
- } else if(f.is_string()) {
- art_tree* t = search_index.at(a_filter.field_name);
+ uint32_t* out = nullptr;
+ result_ids_len = ArrayUtils::or_scalar(&exact_geo_result_ids[0], exact_geo_result_ids.size(),
+ result_ids, result_ids_len, &out);
- uint32_t* or_ids = nullptr;
- size_t or_ids_size = 0;
+ delete[] result_ids;
+ result_ids = out;
- // aggregates IDs across array of filter values and reduces excessive ORing
- std::vector<uint32_t> f_id_buff;
+ delete query_region;
+ }
+ } else if (f.is_string()) {
+ art_tree* t = search_index.at(a_filter.field_name);
- for(const std::string & filter_value: a_filter.values) {
- std::vector<void*> posting_lists;
+ uint32_t* or_ids = nullptr;
+ size_t or_ids_size = 0;
- // there could be multiple tokens in a filter value, which we have to treat as ANDs
- // e.g. country: South Africa
- Tokenizer tokenizer(filter_value, true, false, f.locale, symbols_to_index, token_separators);
+ // aggregates IDs across array of filter values and reduces excessive ORing
+ std::vector<uint32_t> f_id_buff;
- std::string str_token;
- size_t token_index = 0;
- std::vector<std::string> str_tokens;
+ for (const std::string& filter_value : a_filter.values) {
+ std::vector<void*> posting_lists;
- while(tokenizer.next(str_token, token_index)) {
- str_tokens.push_back(str_token);
+ // there could be multiple tokens in a filter value, which we have to treat as ANDs
+ // e.g. country: South Africa
+ Tokenizer tokenizer(filter_value, true, false, f.locale, symbols_to_index, token_separators);
- art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
- str_token.length()+1);
- if(leaf == nullptr) {
- continue;
- }
+ std::string str_token;
+ size_t token_index = 0;
+ std::vector<std::string> str_tokens;
- posting_lists.push_back(leaf->values);
- }
+ while (tokenizer.next(str_token, token_index)) {
+ str_tokens.push_back(str_token);
- if(posting_lists.size() != str_tokens.size()) {
+ art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
+ str_token.length()+1);
+ if (leaf == nullptr) {
continue;
}
- if(a_filter.comparators[0] == EQUALS || a_filter.comparators[0] == NOT_EQUALS) {
- // needs intersection + exact matching (unlike CONTAINS)
- std::vector<uint32_t> result_id_vec;
- posting_t::intersect(posting_lists, result_id_vec);
+ posting_lists.push_back(leaf->values);
+ }
- if(result_id_vec.empty()) {
- continue;
- }
+ if (posting_lists.size() != str_tokens.size()) {
+ continue;
+ }
- // need to do exact match
- uint32_t* exact_str_ids = new uint32_t[result_id_vec.size()];
- size_t exact_str_ids_size = 0;
- std::unique_ptr<uint32_t[]> exact_str_ids_guard(exact_str_ids);
+ if(a_filter.comparators[0] == EQUALS || a_filter.comparators[0] == NOT_EQUALS) {
+ // needs intersection + exact matching (unlike CONTAINS)
+ std::vector<uint32_t> result_id_vec;
+ posting_t::intersect(posting_lists, result_id_vec);
- posting_t::get_exact_matches(posting_lists, f.is_array(), result_id_vec.data(), result_id_vec.size(),
- exact_str_ids, exact_str_ids_size);
+ if (result_id_vec.empty()) {
+ continue;
+ }
- if(exact_str_ids_size == 0) {
- continue;
- }
+ // need to do exact match
+ uint32_t* exact_str_ids = new uint32_t[result_id_vec.size()];
+ size_t exact_str_ids_size = 0;
+ std::unique_ptr<uint32_t[]> exact_str_ids_guard(exact_str_ids);
- for(size_t ei = 0; ei < exact_str_ids_size; ei++) {
- f_id_buff.push_back(exact_str_ids[ei]);
- }
+ posting_t::get_exact_matches(posting_lists, f.is_array(), result_id_vec.data(), result_id_vec.size(),
+ exact_str_ids, exact_str_ids_size);
- } else {
- // CONTAINS
- size_t before_size = f_id_buff.size();
- posting_t::intersect(posting_lists, f_id_buff);
- if(f_id_buff.size() == before_size) {
- continue;
- }
+ if (exact_str_ids_size == 0) {
+ continue;
}
- if(f_id_buff.size() > 100000 || a_filter.values.size() == 1) {
- gfx::timsort(f_id_buff.begin(), f_id_buff.end());
- f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());
-
- uint32_t* out = nullptr;
- or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);
- delete[] or_ids;
- or_ids = out;
- std::vector<uint32_t>().swap(f_id_buff); // clears out memory
+ for (size_t ei = 0; ei < exact_str_ids_size; ei++) {
+ f_id_buff.push_back(exact_str_ids[ei]);
+ }
+ } else {
+ // CONTAINS
+ size_t before_size = f_id_buff.size();
+ posting_t::intersect(posting_lists, f_id_buff);
+ if (f_id_buff.size() == before_size) {
+ continue;
}
}
- if(!f_id_buff.empty()) {
+ if (f_id_buff.size() > 100000 || a_filter.values.size() == 1) {
gfx::timsort(f_id_buff.begin(), f_id_buff.end());
f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());
@@ -1838,77 +1807,120 @@ void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,
or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);
delete[] or_ids;
or_ids = out;
- std::vector<uint32_t>().swap(f_id_buff); // clears out memory
+ std::vector<uint32_t>().swap(f_id_buff); // clears out memory
}
+ }
- if(a_filter.comparators[0] == NOT_EQUALS) {
- // exclude records from existing IDs (from previous filters or ALL records)
- // "not equals" can only be applied to the entire array so we can do the exclusion operations once here
- uint32_t* excluded_strt_ids = nullptr;
- size_t excluded_strt_size = 0;
+ if (!f_id_buff.empty()) {
+ gfx::timsort(f_id_buff.begin(), f_id_buff.end());
+ f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());
- if(result_ids == nullptr) {
- if(filter_ids == nullptr) {
- result_ids = seq_ids->uncompress();
- result_ids_len = seq_ids->num_ids();
- } else {
- result_ids = filter_ids;
- result_ids_len = filter_ids_length;
- }
- }
+ uint32_t* out = nullptr;
+ or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);
+ delete[] or_ids;
+ or_ids = out;
+ std::vector<uint32_t>().swap(f_id_buff); // clears out memory
+ }
- excluded_strt_size = ArrayUtils::exclude_scalar(result_ids, result_ids_len, or_ids,
- or_ids_size, &excluded_strt_ids);
+ if (a_filter.comparators[0] == NOT_EQUALS) {
+ // exclude records from existing IDs (from previous filters or ALL records)
+ // "not equals" can only be applied to the entire array so we can do the exclusion operations once here
+ uint32_t* excluded_strt_ids = nullptr;
+ size_t excluded_strt_size = 0;
- if(filter_ids == nullptr) {
- // means we had to uncompress `seq_ids` so need to free that
- delete [] result_ids;
+ if (result_ids == nullptr) {
+ if (filter_ids == nullptr) {
+ result_ids = seq_ids->uncompress();
+ result_ids_len = seq_ids->num_ids();
+ } else {
+ result_ids = filter_ids;
+ result_ids_len = filter_ids_length;
}
+ }
+
+ excluded_strt_size = ArrayUtils::exclude_scalar(result_ids, result_ids_len, or_ids,
+ or_ids_size, &excluded_strt_ids);
- delete [] or_ids;
- or_ids = excluded_strt_ids;
- or_ids_size = excluded_strt_size;
+ if (filter_ids == nullptr) {
+ // means we had to uncompress `seq_ids` so need to free that
+ delete[] result_ids;
}
- result_ids = or_ids;
- result_ids_len = or_ids_size;
+ delete[] or_ids;
+ or_ids = excluded_strt_ids;
+ or_ids_size = excluded_strt_size;
}
- if(i == 0) {
- filter_ids = result_ids;
- filter_ids_length = result_ids_len;
- } else {
- uint32_t* filtered_results = nullptr;
- filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,
- result_ids_len, &filtered_results);
- delete [] result_ids;
- delete [] filter_ids;
- filter_ids = filtered_results;
- }
+ result_ids = or_ids;
+ result_ids_len = or_ids_size;
}
- if(filter_ids_length == 0) {
- delete [] filter_ids;
- filter_ids = nullptr;
- }
+ filter_ids = result_ids;
+ filter_ids_length = result_ids_len;
/*long long int timeMillis =
- std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+ std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now()
+ - begin).count();
LOG(INFO) << "Time taken for filtering: " << timeMillis << "ms";*/
}
+void Index::recursive_filter(uint32_t*& filter_ids,
+ uint32_t& filter_ids_length,
+ const filter_node_t* root,
+ const bool enable_short_circuit) const {
+ if (root == nullptr) {
+ return;
+ }
+
+ uint32_t* l_filter_ids = nullptr;
+ uint32_t l_filter_ids_length = 0;
+ if (root->left != nullptr) {
+ recursive_filter(l_filter_ids, l_filter_ids_length, root->left,
+ enable_short_circuit);
+ }
-void Index::do_filtering_with_lock(uint32_t*& filter_ids, uint32_t& filter_ids_length,
- const std::vector<filter>& filters) const {
+ uint32_t* r_filter_ids = nullptr;
+ uint32_t r_filter_ids_length = 0;
+ if (root->right != nullptr) {
+ recursive_filter(r_filter_ids, r_filter_ids_length, root->right,
+ enable_short_circuit);
+ }
+
+ if (root->isOperator) {
+ uint32_t* filtered_results = nullptr;
+ if (root->filter_operator == AND) {
+ filter_ids_length = ArrayUtils::and_scalar(
+ l_filter_ids, l_filter_ids_length, r_filter_ids,
+ r_filter_ids_length, &filtered_results);
+ } else {
+ filter_ids_length = ArrayUtils::or_scalar(
+ l_filter_ids, l_filter_ids_length, r_filter_ids,
+ r_filter_ids_length, &filtered_results);
+ }
+
+ delete[] l_filter_ids;
+ delete[] r_filter_ids;
+
+ filter_ids = filtered_results;
+ } else if (root->left == nullptr && root->right == nullptr) {
+ do_filtering(filter_ids, filter_ids_length, root);
+ } else {
+ // malformed
+ }
+}
+
+void Index::do_filtering_with_lock(uint32_t*& filter_ids,
+ uint32_t& filter_ids_length,
+ filter_node_t const* const& filter_tree_root) const {
std::shared_lock lock(mutex);
- do_filtering(filter_ids, filter_ids_length, filters, false);
+ recursive_filter(filter_ids, filter_ids_length, filter_tree_root, false);
}
void Index::run_search(search_args* search_params) {
search(search_params->field_query_tokens,
search_params->search_fields,
- search_params->filters, search_params->facets, search_params->facet_query,
+ search_params->filter_tree_root, search_params->facets, search_params->facet_query,
search_params->included_ids, search_params->excluded_ids,
search_params->sort_fields_std, search_params->num_typos,
search_params->topster, search_params->curated_topster,
@@ -1965,6 +1977,7 @@ void Index::collate_included_ids(const std::vector<token_t>& q_included_tokens,
scores[2] = int64_t(1);
uint32_t token_bits = 0;
+
KV kv(0, searched_queries.size(), token_bits, seq_id, distinct_id, 0, scores);
curated_topster->add(&kv);
}
@@ -1990,17 +2003,27 @@ void Index::concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t,
bool Index::static_filter_query_eval(const override_t* override,
std::vector<std::string>& tokens,
- std::vector<filter>& filters) const {
-
+ filter_node_t*& filter_tree_root) const {
std::string query = StringUtils::join(tokens, " ");
if ((override->rule.match == override_t::MATCH_EXACT && override->rule.query == query) ||
(override->rule.match == override_t::MATCH_CONTAINS &&
StringUtils::contains_word(query, override->rule.query))) {
-
+ filter_node_t* new_filter_tree_root;
Option<bool> filter_op = filter::parse_filter_query(override->filter_by, search_schema,
- store, "", filters);
- return filter_op.ok();
+ store, "", new_filter_tree_root);
+ if (filter_op.ok()) {
+ if (filter_tree_root == nullptr) {
+ filter_tree_root = new_filter_tree_root;
+ } else {
+ filter_node_t* root = new filter_node_t(AND, filter_tree_root,
+ new_filter_tree_root);
+ filter_tree_root = root;
+ }
+ return true;
+ } else {
+ delete new_filter_tree_root;
+ }
}
return false;
@@ -2101,27 +2124,25 @@ bool Index::resolve_override(const std::vector<std::string>& rule_tokens, const
void Index::process_filter_overrides(const std::vector<const override_t*>& filter_overrides,
std::vector<std::string>& query_tokens,
token_ordering token_order,
- std::vector<filter>& filters,
+ filter_node_t*& filter_tree_root,
std::vector<const override_t*>& matched_dynamic_overrides) const {
-
std::shared_lock lock(mutex);
- size_t orig_filters_size = filters.size();
- for(auto& override: filter_overrides) {
- if(!override->rule.dynamic_query) {
+ for (auto& override : filter_overrides) {
+ if (!override->rule.dynamic_query) {
// Simple static filtering: add to filter_by and rewrite query if needed.
// Check the original query and then the synonym variants until a rule matches.
- bool resolved_override = static_filter_query_eval(override, query_tokens, filters);
+ bool resolved_override = static_filter_query_eval(override, query_tokens, filter_tree_root);
- if(resolved_override) {
- if(override->remove_matched_tokens) {
+ if (resolved_override) {
+ if (override->remove_matched_tokens) {
std::vector<std::string> rule_tokens;
Tokenizer(override->rule.query, true).tokenize(rule_tokens);
std::set<std::string> rule_token_set(rule_tokens.begin(), rule_tokens.end());
remove_matched_tokens(query_tokens, rule_token_set);
}
- if(override->stop_processing) {
+ if (override->stop_processing) {
return;
}
}
@@ -2142,20 +2163,31 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filte
bool resolved_override = resolve_override(rule_parts, exact_rule_match, query_tokens,
token_order, absorbed_tokens, filter_by_clause);
- if(resolved_override) {
- Option<bool> filter_parse_op = filter::parse_filter_query(filter_by_clause, search_schema, store, "",
- filters);
- if(filter_parse_op.ok()) {
+ if (resolved_override) {
+ filter_node_t* new_filter_tree_root;
+ Option<bool> filter_op = filter::parse_filter_query(filter_by_clause, search_schema,
+ store, "", new_filter_tree_root);
+ if (filter_op.ok()) {
// have to ensure that dropped hits take precedence over added hits
matched_dynamic_overrides.push_back(override);
- if(override->remove_matched_tokens) {
+ if (override->remove_matched_tokens) {
std::vector<std::string>& tokens = query_tokens;
remove_matched_tokens(tokens, absorbed_tokens);
}
+
+ if (filter_tree_root == nullptr) {
+ filter_tree_root = new_filter_tree_root;
+ } else {
+ filter_node_t* root = new filter_node_t(AND, filter_tree_root,
+ new_filter_tree_root);
+ filter_tree_root = root;
+ }
+ } else {
+ delete new_filter_tree_root;
}
- if(override->stop_processing) {
+ if (override->stop_processing) {
return;
}
}
@@ -2292,8 +2324,8 @@ void Index::search_infix(const std::string& query, const std::string& field_name
for(auto infix_set: infix_sets) {
thread_pool->enqueue([infix_set, &leaves, search_tree, &query, max_extra_prefix, max_extra_suffix,
- &num_processed, &m_process, &cv_process,
- &parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff]() {
+ &num_processed, &m_process, &cv_process,
+ &parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff]() {
search_begin = parent_search_begin;
search_cutoff = parent_search_cutoff;
@@ -2346,7 +2378,7 @@ void Index::search_infix(const std::string& query, const std::string& field_name
}
void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vector<search_field_t>& the_fields,
- std::vector<filter>& filters, std::vector<facet>& facets, facet_query_t& facet_query,
+ filter_node_t const* const& filter_tree_root, std::vector<facet>& facets, facet_query_t& facet_query,
const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
const std::vector<uint32_t>& excluded_ids, std::vector<sort_by>& sort_fields_std,
const std::vector<uint32_t>& num_typos, Topster* topster, Topster* curated_topster,
@@ -2374,10 +2406,11 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
std::shared_lock lock(mutex);
- do_filtering(filter_ids, filter_ids_length, filters, true);
+ recursive_filter(filter_ids, filter_ids_length, filter_tree_root, true);
- if(!filters.empty() && filter_ids_length == 0) {
- return ;
+ if (filter_tree_root != nullptr && filter_ids_length == 0) {
+ delete [] filter_ids;
+ return;
}
std::set<uint32_t> curated_ids;
@@ -2390,16 +2423,16 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
std::sort(curated_ids_sorted.begin(), curated_ids_sorted.end());
// Order of `fields` are used to sort results
- //auto begin = std::chrono::high_resolution_clock::now();
+ // auto begin = std::chrono::high_resolution_clock::now();
uint32_t* all_result_ids = nullptr;
const size_t num_search_fields = std::min(the_fields.size(), (size_t) FIELD_LIMIT_NUM);
// handle phrase searches
- if(!field_query_tokens[0].q_phrases.empty()) {
+ if (!field_query_tokens[0].q_phrases.empty()) {
do_phrase_search(num_search_fields, the_fields, field_query_tokens, filter_ids, filter_ids_length);
- if(filter_ids_length == 0) {
- return ;
+ if (filter_ids_length == 0) {
+ return;
}
}
@@ -2408,7 +2441,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
size_t exclude_token_ids_size = 0;
handle_exclusion(num_search_fields, field_query_tokens, the_fields, exclude_token_ids, exclude_token_ids_size);
- int sort_order[3]; // 1 or -1 based on DESC or ASC respectively
+ int sort_order[3]; // 1 or -1 based on DESC or ASC respectively
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values;
std::vector<size_t> geopoint_indices;
populate_sort_mapping(sort_order, geopoint_indices, sort_fields_std, field_values);
@@ -2425,7 +2458,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
// for phrase query, parser will set field_query_tokens to "*", need to handle that
if (is_wildcard_query) {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
- bool no_filters_provided = (filters.empty() && filter_ids_length == 0);
+ bool no_filters_provided =
+ (filter_tree_root != nullptr && filter_ids_length == 0);
if(no_filters_provided && facets.empty() && curated_ids.empty() && vector_query.field_name.empty() &&
sort_fields_std.size() == 1 && sort_fields_std[0].name == sort_field_const::seq_id &&
@@ -2433,10 +2467,10 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
// optimize for this path specifically
std::vector<uint32_t> result_ids;
auto it = seq_ids->new_rev_iterator();
- while(it.valid()) {
+ while (it.valid()) {
uint32_t seq_id = it.id();
uint64_t distinct_id = seq_id;
- if(group_limit != 0) {
+ if (group_limit != 0) {
distinct_id = get_distinct_id(group_by_fields, seq_id);
groups_processed.emplace(distinct_id);
}
@@ -2449,7 +2483,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
KV kv(field_id, searched_queries.size(), 0, seq_id, distinct_id, match_score_index, scores);
topster->add(&kv);
- if(result_ids.size() == page * per_page) {
+ if (result_ids.size() == page * per_page) {
break;
}
@@ -2460,17 +2494,18 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
goto process_search_results;
}
- // if filters were not provided, use the seq_ids index to generate the list of all document ids
- if(no_filters_provided) {
+ // if filters were not provided, use the seq_ids index to generate the
+ // list of all document ids
+ if (no_filters_provided) {
filter_ids_length = seq_ids->num_ids();
filter_ids = seq_ids->uncompress();
}
- curate_filtered_ids(filters, curated_ids, excluded_result_ids,
+ curate_filtered_ids(filter_tree_root, curated_ids, excluded_result_ids,
excluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted);
collate_included_ids({}, included_ids_map, curated_topster, searched_queries);
- if(!vector_query.field_name.empty()) {
+ if (!vector_query.field_name.empty()) {
auto k = std::max<size_t>(vector_query.k, per_page * page);
VectorFilterFunctor filterFunctor(filter_ids, filter_ids_length);
@@ -2478,6 +2513,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
std::vector<std::pair<float, size_t>> dist_labels;
+
if(field_vector_index->distance_type == cosine) {
std::vector<float> normalized_q(vector_query.values.size());
hnsw_index_t::normalize_vector(vector_query.values, normalized_q);
@@ -2488,10 +2524,10 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
std::vector<uint32_t> nearest_ids;
- for(const auto& dist_label: dist_labels) {
+ for (const auto& dist_label : dist_labels) {
uint32 seq_id = dist_label.second;
uint64_t distinct_id = seq_id;
- if(group_limit != 0) {
+ if (group_limit != 0) {
distinct_id = get_distinct_id(group_by_fields, seq_id);
groups_processed.emplace(distinct_id);
}
@@ -2517,9 +2553,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
delete [] all_result_ids;
all_result_ids = new_all_result_ids;
}
-
} else {
- search_wildcard(filters, included_ids_map, sort_fields_std, topster,
+ search_wildcard(filter_tree_root, included_ids_map, sort_fields_std, topster,
curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,
curated_ids, curated_ids_sorted,
excluded_result_ids, excluded_result_ids_size,
@@ -2577,7 +2612,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
if(split_join_tokens == always || (all_result_ids_len == 0 && split_join_tokens == fallback)) {
std::vector<std::vector<std::string>> space_resolved_queries;
- for(size_t i = 0; i < num_search_fields; i++) {
+ for (size_t i = 0; i < num_search_fields; i++) {
std::vector<std::string> orig_q_include_tokens;
for(auto& q_include_token: field_query_tokens[i].q_include_tokens) {
orig_q_include_tokens.push_back(q_include_token.value);
@@ -2585,20 +2620,20 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
resolve_space_as_typos(orig_q_include_tokens, the_fields[i].name,space_resolved_queries);
- if(!space_resolved_queries.empty()) {
+ if (!space_resolved_queries.empty()) {
break;
}
}
// only one query is resolved for now, so just use that
- if(!space_resolved_queries.empty()) {
+ if (!space_resolved_queries.empty()) {
const auto& resolved_query = space_resolved_queries[0];
std::vector<token_t> resolved_tokens;
for(size_t j=0; j < resolved_query.size(); j++) {
bool is_prefix = (j == resolved_query.size()-1);
resolved_tokens.emplace_back(j, space_resolved_queries[0][j], is_prefix,
- space_resolved_queries[0][j].size(), 0);
+ space_resolved_queries[0][j].size(), 0);
}
fuzzy_search_fields(the_fields, resolved_tokens, false, excluded_result_ids,
@@ -2611,7 +2646,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
}
// do synonym based searches
- do_synonym_search(the_fields, filters, included_ids_map, sort_fields_std, curated_topster, token_order,
+ do_synonym_search(the_fields, filter_tree_root, included_ids_map, sort_fields_std, curated_topster, token_order,
0, group_limit, group_by_fields, prioritize_exact_match, prioritize_token_position,
exhaustive_search, concurrency, prefixes,
min_len_1typo, min_len_2typo, max_candidates, curated_ids, curated_ids_sorted,
@@ -2623,8 +2658,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
// gather up both original query and synonym queries and do drop tokens
- if(all_result_ids_len < drop_tokens_threshold) {
- for(size_t qi = 0; qi < all_queries.size(); qi++) {
+ if (all_result_ids_len < drop_tokens_threshold) {
+ for (size_t qi = 0; qi < all_queries.size(); qi++) {
auto& orig_tokens = all_queries[qi];
size_t num_tokens_dropped = 0;
@@ -2635,10 +2670,10 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
if(orig_tokens.size() > 1 && num_tokens_dropped < 2*(orig_tokens.size()-1)) {
bool prefix_search = false;
- if(num_tokens_dropped < orig_tokens.size()-1) {
+ if (num_tokens_dropped < orig_tokens.size() - 1) {
// drop from right
size_t truncated_len = orig_tokens.size() - num_tokens_dropped - 1;
- for(size_t i=0; i < truncated_len; i++) {
+ for (size_t i = 0; i < truncated_len; i++) {
truncated_tokens.emplace_back(orig_tokens[i]);
}
} else {
@@ -2653,7 +2688,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
num_tokens_dropped++;
std::vector<bool> drop_token_prefixes;
- for(const auto p: prefixes) {
+ for (const auto p : prefixes) {
drop_token_prefixes.push_back(p && prefix_search);
}
@@ -2681,9 +2716,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
curated_ids_sorted, all_result_ids, all_result_ids_len, groups_processed);
/*auto timeMillis0 = std::chrono::duration_cast<std::chrono::milliseconds>(
- std::chrono::high_resolution_clock::now() - begin0).count();
-
- LOG(INFO) << "Time taken for multi-field aggregation: " << timeMillis0 << "ms";*/
+ std::chrono::high_resolution_clock::now() - begin0).count();
+ LOG(INFO) << "Time taken for multi-field aggregation: " << timeMillis0 << "ms";*/
}
//LOG(INFO) << "topster size: " << topster->size;
@@ -2753,8 +2787,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
if(group_limit) {
// we have to add all group sets
acc_facet.hash_groups[facet_kv.first].insert(
- this_facet.hash_groups[facet_kv.first].begin(),
- this_facet.hash_groups[facet_kv.first].end()
+ this_facet.hash_groups[facet_kv.first].begin(),
+ this_facet.hash_groups[facet_kv.first].end()
);
} else {
size_t count = 0;
@@ -2807,7 +2841,7 @@ void Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>
std::set<uint32_t>& curated_ids,
std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
std::vector<uint32_t>& included_ids_vec) const {
-
+
for(const auto& seq_id_pos: included_ids) {
included_ids_vec.push_back(seq_id_pos.first);
}
@@ -2819,7 +2853,7 @@ void Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>
if(filter_ids_length != 0 && filter_curated_hits) {
uint32_t* included_ids_arr = nullptr;
size_t included_ids_len = ArrayUtils::and_scalar(&included_ids_vec[0], included_ids_vec.size(), filter_ids,
- filter_ids_length, &included_ids_arr);
+ filter_ids_length, &included_ids_arr);
included_ids_vec.clear();
@@ -2994,7 +3028,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
the_fields, num_search_fields, popular_field_ids);
if(popular_field_ids.empty()) {
- break;
+ break;
}
}
@@ -3047,8 +3081,8 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
auto& prev_token = token_candidates_vec.back().candidates[0];
art_leaf* prev_leaf = static_cast<art_leaf*>(
art_search(search_index.at(the_field.name),
- reinterpret_cast<const unsigned char*>(prev_token.c_str()),
- prev_token.size() + 1));
+ reinterpret_cast<const unsigned char*>(prev_token.c_str()),
+ prev_token.size() + 1));
if(!prev_leaf) {
continue;
@@ -3465,10 +3499,10 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
}
score_results2(sort_fields, searched_queries.size(), fi, field_is_array,
- total_cost, field_match_score,
- seq_id, sort_order,
- prioritize_exact_match, single_exact_query_token, prioritize_token_position,
- query_tokens.size(), syn_orig_num_tokens, token_postings);
+ total_cost, field_match_score,
+ seq_id, sort_order,
+ prioritize_exact_match, single_exact_query_token, prioritize_token_position,
+ query_tokens.size(), syn_orig_num_tokens, token_postings);
if(field_match_score > max_field_match_score) {
max_field_match_score = field_match_score;
@@ -3631,7 +3665,7 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i
}
else if(sort_fields[0].order == sort_field_const::desc &&
- sort_fields[0].missing_values == sort_by::missing_values_t::last) {
+ sort_fields[0].missing_values == sort_by::missing_values_t::last) {
scores[0] = -scores[0];
}
}
@@ -3876,7 +3910,7 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<s
}
void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
- const std::vector<filter>& filters,
+ filter_node_t const* const& filter_tree_root,
const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
const token_ordering& token_order,
@@ -3902,7 +3936,7 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
const std::vector<size_t>& geopoint_indices,
tsl::htrie_map<char, token_leaf>& qtoken_set) const {
- for(const auto& syn_tokens: q_pos_synonyms) {
+ for (const auto& syn_tokens : q_pos_synonyms) {
query_hashes.clear();
fuzzy_search_fields(the_fields, syn_tokens, false, exclude_token_ids,
exclude_token_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,
@@ -4206,10 +4240,18 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
}
}
-void Index::curate_filtered_ids(const std::vector<filter>& filters, const std::set<uint32_t>& curated_ids,
+void Index::curate_filtered_ids(filter_node_t const* const& filter_tree_root, const std::set<uint32_t>& curated_ids,
const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
uint32_t*& filter_ids, uint32_t& filter_ids_length,
const std::vector<uint32_t>& curated_ids_sorted) const {
+
+ // if filtered results are not available, use the seq_ids index to generate
+ // the list of all document ids
+ if (filter_tree_root == nullptr && filter_ids_length == 0) {
+ filter_ids_length = seq_ids->num_ids();
+ filter_ids = seq_ids->uncompress();
+ }
+
if(!curated_ids.empty()) {
uint32_t *excluded_result_ids = nullptr;
filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, &curated_ids_sorted[0],
@@ -4228,7 +4270,7 @@ void Index::curate_filtered_ids(const std::vector<filter>& filters, const std::s
}
}
-void Index::search_wildcard(const std::vector<filter>& filters,
+void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
spp::sparse_hash_set<uint64_t>& groups_processed,
@@ -4281,12 +4323,12 @@ void Index::search_wildcard(const std::vector<filter>& filters,
topsters[thread_id] = new Topster(topster->MAX_SIZE, topster->distinct);
thread_pool->enqueue([this, &parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,
- thread_id, &sort_fields, &searched_queries,
- &group_limit, &group_by_fields, &topsters, &tgroups_processed,
- &sort_order, field_values, &geopoint_indices, &plists,
- check_for_circuit_break,
- batch_result_ids, batch_res_len,
- &num_processed, &m_process, &cv_process]() {
+ thread_id, &sort_fields, &searched_queries,
+ &group_limit, &group_by_fields, &topsters, &tgroups_processed,
+ &sort_order, field_values, &geopoint_indices, &plists,
+ check_for_circuit_break,
+ batch_result_ids, batch_res_len,
+ &num_processed, &m_process, &cv_process]() {
search_begin = parent_search_begin;
search_stop_ms = parent_search_stop_ms;
@@ -4368,7 +4410,7 @@ void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint
field_values[i] = &seq_id_sentinel_value;
} else if (sort_fields_std[i].name == sort_field_const::eval) {
field_values[i] = &eval_sentinel_value;
- do_filtering(sort_fields_std[i].eval.ids, sort_fields_std[i].eval.size, sort_fields_std[i].eval.filters, true);
+ recursive_filter(sort_fields_std[i].eval.ids, sort_fields_std[i].eval.size, sort_fields_std[i].eval.filter_tree_root, true);
} else if (search_schema.count(sort_fields_std[i].name) != 0 && search_schema.at(sort_fields_std[i].name).sort) {
if (search_schema.at(sort_fields_std[i].name).type == field_types::GEOPOINT_ARRAY) {
geopoint_indices.push_back(i);
@@ -4386,15 +4428,6 @@ void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint
}
}
-/*
- 1. Split the query into tokens
- 2. Outer loop will generate bounded cartesian product with costs for each token
- 3. Inner loop will iterate on each token with associated cost
- 4. Cartesian product of the results of the token searches will be used to form search phrases
- (cartesian product adapted from: http://stackoverflow.com/a/31169617/131050)
- 4. Intersect the lists to find docs that match each phrase
- 5. Sort the docs based on some ranking criteria
-*/
void Index::search_field(const uint8_t & field_id,
const std::vector<token_t>& query_tokens,
const uint32_t* exclude_token_ids,
@@ -4513,7 +4546,6 @@ void Index::search_field(const uint8_t & field_id,
/*auto timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
-
LOG(INFO) << "Time taken for fuzzy search: " << timeMillis << "ms";*/
if(!leaves.empty()) {
@@ -4596,25 +4628,25 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vect
}
int64_t Index::score_results2(const std::vector<sort_by> & sort_fields, const uint16_t & query_index,
- const size_t field_id,
- const bool field_is_array,
- const uint32_t total_cost,
- int64_t& match_score,
- const uint32_t seq_id, const int sort_order[3],
- const bool prioritize_exact_match,
- const bool single_exact_query_token,
- const bool prioritize_token_position,
- size_t num_query_tokens,
- int syn_orig_num_tokens,
- const std::vector<posting_list_t::iterator_t>& posting_lists) const {
+ const size_t field_id,
+ const bool field_is_array,
+ const uint32_t total_cost,
+ int64_t& match_score,
+ const uint32_t seq_id, const int sort_order[3],
+ const bool prioritize_exact_match,
+ const bool single_exact_query_token,
+ const bool prioritize_token_position,
+ size_t num_query_tokens,
+ int syn_orig_num_tokens,
+ const std::vector<posting_list_t::iterator_t>& posting_lists) const {
//auto begin = std::chrono::high_resolution_clock::now();
//const std::string first_token((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1);
if (posting_lists.size() <= 1) {
const uint8_t is_verbatim_match = uint8_t(
- prioritize_exact_match && single_exact_query_token &&
- posting_list_t::is_single_token_verbatim_match(posting_lists[0], field_is_array)
+ prioritize_exact_match && single_exact_query_token &&
+ posting_list_t::is_single_token_verbatim_match(posting_lists[0], field_is_array)
);
size_t words_present = (num_query_tokens == 1 && syn_orig_num_tokens != -1) ? syn_orig_num_tokens : 1;
size_t distance = (num_query_tokens == 1 && syn_orig_num_tokens != -1) ? syn_orig_num_tokens-1 : 0;
@@ -4629,7 +4661,6 @@ int64_t Index::score_results2(const std::vector<sort_by> & sort_fields, const ui
auto proximity = ((match_score >> 16) & 0xFF);
auto verbatim = ((match_score >> 8) & 0xFF);
auto offset_score = ((match_score >> 0) & 0xFF);
-
LOG(INFO) << "seq_id: " << seq_id
<< ", words_present: " << this_words_present
<< ", unique_words: " << unique_words
@@ -4669,12 +4700,12 @@ int64_t Index::score_results2(const std::vector<sort_by> & sort_fields, const ui
}
uint64_t mod_match_score = (
- (int64_t(this_words_present) << 40) |
- (int64_t(unique_words) << 32) |
- (int64_t(typo_score) << 24) |
- (int64_t(proximity) << 16) |
- (int64_t(verbatim) << 8) |
- (int64_t(offset_score) << 0)
+ (int64_t(this_words_present) << 40) |
+ (int64_t(unique_words) << 32) |
+ (int64_t(typo_score) << 24) |
+ (int64_t(proximity) << 16) |
+ (int64_t(verbatim) << 8) |
+ (int64_t(offset_score) << 0)
);
if(mod_match_score > match_score) {
@@ -4779,8 +4810,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
if (posting_lists.size() <= 1) {
const uint8_t is_verbatim_match = uint8_t(
- prioritize_exact_match && single_exact_query_token &&
- posting_list_t::is_single_token_verbatim_match(posting_lists[0], field_is_array)
+ prioritize_exact_match && single_exact_query_token &&
+ posting_list_t::is_single_token_verbatim_match(posting_lists[0], field_is_array)
);
size_t words_present = (syn_orig_num_tokens == -1) ? 1 : syn_orig_num_tokens;
size_t distance = (syn_orig_num_tokens == -1) ? 0 : syn_orig_num_tokens-1;
@@ -4817,11 +4848,11 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
}
uint64_t mod_match_score = (
- (int64_t(unique_tokens_found) << 32) |
- (int64_t(this_words_present) << 24) |
- (int64_t(typo_score) << 16) |
- (int64_t(proximity) << 8) |
- (int64_t(verbatim) << 0)
+ (int64_t(unique_tokens_found) << 32) |
+ (int64_t(this_words_present) << 24) |
+ (int64_t(typo_score) << 16) |
+ (int64_t(proximity) << 8) |
+ (int64_t(verbatim) << 0)
);
if(mod_match_score > match_score) {
@@ -4982,12 +5013,12 @@ inline uint32_t Index::next_suggestion2(const std::vector<tok_candidates>& token
}
inline uint32_t Index::next_suggestion(const std::vector<token_candidates> &token_candidates_vec,
- long long int n,
- std::vector<art_leaf *>& actual_query_suggestion,
- std::vector<art_leaf *>& query_suggestion,
- const int syn_orig_num_tokens,
- uint32_t& token_bits,
- uint64& qhash) {
+ long long int n,
+ std::vector<art_leaf *>& actual_query_suggestion,
+ std::vector<art_leaf *>& query_suggestion,
+ const int syn_orig_num_tokens,
+ uint32_t& token_bits,
+ uint64& qhash) {
uint32_t total_cost = 0;
qhash = 1;
@@ -5027,7 +5058,7 @@ inline uint32_t Index::next_suggestion(const std::vector<token_candidates> &toke
}
void Index::remove_facet_token(const field& search_field, spp::sparse_hash_map<std::string, art_tree*>& search_index,
- const std::string& token, uint32_t seq_id) {
+ const std::string& token, uint32_t seq_id) {
const unsigned char *key = (const unsigned char *) token.c_str();
int key_len = (int) (token.length() + 1);
const std::string& field_name = search_field.faceted_name();
@@ -6020,52 +6051,41 @@ bool Index::common_results_exist(std::vector<art_leaf*>& leaves, bool must_match
bool Index::is_point_in_polygon(const Geofence& poly, const GeoCoord &point) {
int i, j;
bool c = false;
-
for (i = 0, j = poly.numVerts - 1; i < poly.numVerts; j = i++) {
if ((((poly.verts[i].lat <= point.lat) && (point.lat < poly.verts[j].lat))
|| ((poly.verts[j].lat <= point.lat) && (point.lat < poly.verts[i].lat)))
&& (point.lon < (poly.verts[j].lon - poly.verts[i].lon) * (point.lat - poly.verts[i].lat)
/ (poly.verts[j].lat - poly.verts[i].lat) + poly.verts[i].lon)) {
-
c = !c;
}
}
-
return c;
}
-
double Index::transform_for_180th_meridian(Geofence &poly) {
double offset = 0.0;
double maxLon = -1000, minLon = 1000;
-
for(int v=0; v < poly.numVerts; v++) {
if(poly.verts[v].lon < minLon) {
minLon = poly.verts[v].lon;
}
-
if(poly.verts[v].lon > maxLon) {
maxLon = poly.verts[v].lon;
}
-
if(std::abs(minLon - maxLon) > 180) {
offset = 360.0;
}
}
-
int i, j;
for (i = 0, j = poly.numVerts - 1; i < poly.numVerts; j = i++) {
if (poly.verts[i].lon < 0.0) {
poly.verts[i].lon += offset;
}
-
if (poly.verts[j].lon < 0.0) {
poly.verts[j].lon += offset;
}
}
-
return offset;
}
-
void Index::transform_for_180th_meridian(GeoCoord &point, double offset) {
point.lon = point.lon < 0.0 ? point.lon + offset : point.lon;
}
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
index baaa828d..62ee8a24 100644
--- a/src/string_utils.cpp
+++ b/src/string_utils.cpp
@@ -9,7 +9,6 @@
#include <arpa/inet.h>
#include "logger.h"
-
StringUtils::StringUtils() {
UErrorCode errcode = U_ZERO_ERROR;
nfkd = icu::Normalizer2::getNFKDInstance(errcode);
@@ -346,6 +345,67 @@ size_t StringUtils::get_num_chars(const std::string& s) {
return j;
}
+Option<bool> StringUtils::tokenize(std::string filter_query, std::queue<std::string>& tokens) {
+ auto size = filter_query.size();
+ for (auto i = 0; i < size;) {
+ auto c = filter_query[i];
+ if (c == ' ') {
+ i++;
+ continue;
+ }
+
+ if (c == '(') {
+ tokens.push("(");
+ i++;
+ } else if (c == ')') {
+ tokens.push(")");
+ i++;
+ } else if (c == '&') {
+ if (i + 1 >= size || filter_query[i + 1] != '&') {
+ return Option<bool>(400, "Could not parse the filter filter_query.");
+ }
+ tokens.push("&&");
+ i += 2;
+ } else if (c == '|') {
+ if (i + 1 >= size || filter_query[i + 1] != '|') {
+ return Option<bool>(400, "Could not parse the filter filter_query.");
+ }
+ tokens.push("||");
+ i += 2;
+ } else {
+ std::stringstream ss;
+ bool inBacktick = false;
+ bool preceding_colon = false;
+ bool is_geo_value = false;
+
+ do {
+ if (c == ':') {
+ preceding_colon = true;
+ }
+ if (c == ')' && is_geo_value) {
+ is_geo_value = false;
+ }
+
+ ss << c;
+ c = filter_query[++i];
+
+ if (c == '`') {
+ inBacktick = !inBacktick;
+ }
+ if (preceding_colon && c == '(') {
+ is_geo_value = true;
+ preceding_colon = false;
+ } else if (preceding_colon && c != ' ') {
+ preceding_colon = false;
+ }
+ } while (i < size && (inBacktick || is_geo_value ||
+ (c != '(' && c != ')' && !(c == '&' && filter_query[i + 1] == '&') &&
+ !(c == '|' && filter_query[i + 1] == '|'))));
+ tokens.push(ss.str());
+ }
+ }
+ return Option<bool>(true);
+}
/*size_t StringUtils::unicode_length(const std::string& bytes) {
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8conv;