Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor filtering logic to overcome circular referencing. Handle exact string filtering in `filter_result_iterator`.

Harpreet Sangar7b3b321aafa3
diff --git a/include/art.h b/include/art.h
index a9715fac..11f57a68 100644
--- a/include/art.h
+++ b/include/art.h
@@ -1,5 +1,4 @@
-#ifndef ART_H
-#define ART_H
+#pragma once
#include <stdint.h>
#include <stdbool.h>
@@ -7,6 +6,7 @@
#include <set>
#include "array.h"
#include "sorted_array.h"
+#include "filter_result_iterator.h"
#define IGNORE_PRINTF 1
@@ -111,7 +111,7 @@ struct token_leaf {
uint32_t num_typos;
token_leaf(art_leaf* leaf, uint32_t root_len, uint32_t num_typos, bool is_prefix) :
- leaf(leaf), root_len(root_len), num_typos(num_typos), is_prefix(is_prefix) {
+ leaf(leaf), root_len(root_len), num_typos(num_typos), is_prefix(is_prefix) {
}
};
@@ -157,11 +157,6 @@ enum NUM_COMPARATOR {
RANGE_INCLUSIVE
};
-enum FILTER_OPERATOR {
- AND,
- OR
-};
-
/**
* Initializes an ART tree
* @return 0 on success.
@@ -281,6 +276,12 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
const uint32_t *filter_ids, const size_t filter_ids_length,
std::vector<art_leaf *> &results, std::set<std::string>& exclude_leaves);
+int art_fuzzy_search_i(art_tree *t, const unsigned char *term, const int term_len, const int min_cost, const int max_cost,
+ const size_t max_words, const token_ordering token_order,
+ const bool prefix, bool last_token, const std::string& prev_token,
+ filter_result_iterator_t& filter_result_iterator,
+ std::vector<art_leaf *> &results, std::set<std::string>& exclude_leaves);
+
void encode_int32(int32_t n, unsigned char *chars);
void encode_int64(int64_t n, unsigned char *chars);
@@ -295,6 +296,4 @@ int art_float_search(art_tree *t, float value, NUM_COMPARATOR comparator, std::v
#ifdef __cplusplus
}
-#endif
-
#endif
\ No newline at end of file
diff --git a/include/field.h b/include/field.h
index 8d1606e3..e2f9729f 100644
--- a/include/field.h
+++ b/include/field.h
@@ -2,13 +2,13 @@
#include <string>
#include <s2/s2latlng.h>
-#include "art.h"
#include "option.h"
#include "string_utils.h"
#include "logger.h"
#include "store.h"
#include <sparsepp.h>
#include <tsl/htrie_map.h>
+#include <filter.h>
#include "json.hpp"
#include "text_embedder_manager.h"
@@ -515,200 +515,19 @@ struct field {
static void compact_nested_fields(tsl::htrie_map<char, field>& nested_fields);
};
-struct filter_node_t;
-
-struct filter {
- std::string field_name;
- std::vector<std::string> values;
- std::vector<NUM_COMPARATOR> comparators;
- // Would be set when `field: != ...` is encountered with a string field or `field: != [ ... ]` is encountered in the
- // case of int and float fields. During filtering, all the results of matching the field against the values are
- // aggregated and then this flag is checked if negation on the aggregated result is required.
- bool apply_not_equals = false;
-
- // Would store `Foo` in case of a filter expression like `$Foo(bar := baz)`
- std::string referenced_collection_name = "";
-
- static const std::string RANGE_OPERATOR() {
- return "..";
- }
-
- static Option<bool> validate_numerical_filter_value(field _field, const std::string& raw_value) {
- if(_field.is_int32() && !StringUtils::is_int32_t(raw_value)) {
- return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not an int32.");
- }
-
- else if(_field.is_int64() && !StringUtils::is_int64_t(raw_value)) {
- return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not an int64.");
- }
-
- else if(_field.is_float() && !StringUtils::is_float(raw_value)) {
- return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not a float.");
- }
-
- return Option<bool>(true);
- }
-
- static Option<NUM_COMPARATOR> extract_num_comparator(std::string & comp_and_value) {
- auto num_comparator = EQUALS;
-
- if(StringUtils::is_integer(comp_and_value) || StringUtils::is_float(comp_and_value)) {
- num_comparator = EQUALS;
- }
-
- // the ordering is important - we have to compare 2-letter operators first
- else if(comp_and_value.compare(0, 2, "<=") == 0) {
- num_comparator = LESS_THAN_EQUALS;
- }
-
- else if(comp_and_value.compare(0, 2, ">=") == 0) {
- num_comparator = GREATER_THAN_EQUALS;
- }
-
- else if(comp_and_value.compare(0, 2, "!=") == 0) {
- num_comparator = NOT_EQUALS;
- }
-
- else if(comp_and_value.compare(0, 1, "<") == 0) {
- num_comparator = LESS_THAN;
- }
-
- else if(comp_and_value.compare(0, 1, ">") == 0) {
- num_comparator = GREATER_THAN;
- }
-
- else if(comp_and_value.find("..") != std::string::npos) {
- num_comparator = RANGE_INCLUSIVE;
- }
-
- else {
- return Option<NUM_COMPARATOR>(400, "Numerical field has an invalid comparator.");
- }
-
- if(num_comparator == LESS_THAN || num_comparator == GREATER_THAN) {
- comp_and_value = comp_and_value.substr(1);
- } else if(num_comparator == LESS_THAN_EQUALS || num_comparator == GREATER_THAN_EQUALS || num_comparator == NOT_EQUALS) {
- comp_and_value = comp_and_value.substr(2);
- }
-
- comp_and_value = StringUtils::trim(comp_and_value);
-
- return Option<NUM_COMPARATOR>(num_comparator);
- }
-
- static Option<bool> parse_geopoint_filter_value(std::string& raw_value,
- const std::string& format_err_msg,
- std::string& processed_filter_val,
- NUM_COMPARATOR& num_comparator);
-
- static Option<bool> parse_filter_query(const std::string& filter_query,
- const tsl::htrie_map<char, field>& search_schema,
- const Store* store,
- const std::string& doc_id_prefix,
- filter_node_t*& root);
+enum index_operation_t {
+ CREATE,
+ UPSERT,
+ UPDATE,
+ EMPLACE,
+ DELETE
};
-struct filter_node_t {
- filter filter_exp;
- FILTER_OPERATOR filter_operator;
- bool isOperator;
- filter_node_t* left = nullptr;
- filter_node_t* right = nullptr;
-
- filter_node_t(filter filter_exp)
- : filter_exp(std::move(filter_exp)),
- isOperator(false),
- left(nullptr),
- right(nullptr) {}
-
- filter_node_t(FILTER_OPERATOR filter_operator,
- filter_node_t* left,
- filter_node_t* right)
- : filter_operator(filter_operator),
- isOperator(true),
- left(left),
- right(right) {}
-
- ~filter_node_t() {
- delete left;
- delete right;
- }
-};
-
-struct reference_filter_result_t {
- uint32_t count = 0;
- uint32_t* docs = nullptr;
-
- reference_filter_result_t& operator=(const reference_filter_result_t& obj) noexcept {
- if (&obj == this)
- return *this;
-
- count = obj.count;
- docs = new uint32_t[count];
- memcpy(docs, obj.docs, count * sizeof(uint32_t));
-
- return *this;
- }
-
- ~reference_filter_result_t() {
- delete[] docs;
- }
-};
-
-struct filter_result_t {
- uint32_t count = 0;
- uint32_t* docs = nullptr;
- // Collection name -> Reference filter result
- std::map<std::string, reference_filter_result_t*> reference_filter_results;
-
- filter_result_t() = default;
-
- filter_result_t(uint32_t count, uint32_t* docs) : count(count), docs(docs) {}
-
- filter_result_t& operator=(const filter_result_t& obj) noexcept {
- if (&obj == this)
- return *this;
-
- count = obj.count;
- docs = new uint32_t[count];
- memcpy(docs, obj.docs, count * sizeof(uint32_t));
-
- // Copy every collection's references.
- for (const auto &item: obj.reference_filter_results) {
- reference_filter_results[item.first] = new reference_filter_result_t[count];
-
- for (uint32_t i = 0; i < count; i++) {
- reference_filter_results[item.first][i] = item.second[i];
- }
- }
-
- return *this;
- }
-
- filter_result_t& operator=(filter_result_t&& obj) noexcept {
- if (&obj == this)
- return *this;
-
- count = obj.count;
- docs = obj.docs;
- reference_filter_results = std::map(obj.reference_filter_results);
-
- obj.docs = nullptr;
- obj.reference_filter_results.clear();
-
- return *this;
- }
-
- ~filter_result_t() {
- delete[] docs;
- for (const auto &item: reference_filter_results) {
- delete[] item.second;
- }
- }
-
- static void and_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result);
-
- static void or_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result);
+enum class DIRTY_VALUES {
+ REJECT = 1,
+ DROP = 2,
+ COERCE_OR_REJECT = 3,
+ COERCE_OR_DROP = 4,
};
namespace sort_field_const {
diff --git a/include/filter.h b/include/filter.h
index 4103f49f..1961b0ac 100644
--- a/include/filter.h
+++ b/include/filter.h
@@ -2,155 +2,73 @@
#include <string>
#include <map>
-#include <field.h>
-#include "posting_list.h"
-#include "index.h"
+#include <tsl/htrie_map.h>
+#include <art.h>
+#include "store.h"
-class Index;
-struct filter_node_t;
-struct reference_filter_result_t;
-struct filter_result_t;
-
-class filter_result_iterator_t {
-private:
- std::string collection_name;
- const Index* index = nullptr;
- const filter_node_t* filter_node = nullptr;
- filter_result_iterator_t* left_it = nullptr;
- filter_result_iterator_t* right_it = nullptr;
-
- /// Used in case of id and reference filter.
- uint32_t result_index = 0;
-
- /// Stores the result of the filters that cannot be iterated.
- filter_result_t filter_result;
-
- /// Initialized in case of filter on string field.
- /// Sample filter values: ["foo bar", "baz"]. Each filter value is split into tokens. We get posting list iterator
- /// for each token.
- ///
- /// Multiple filter values: Multiple tokens: posting list iterator
- std::vector<std::vector<posting_list_t::iterator_t>> posting_list_iterators;
- std::vector<posting_list_t*> expanded_plists;
-
- /// Set to false when this iterator or it's subtree becomes invalid.
- bool is_valid = true;
-
- /// Initializes the state of iterator node after it's creation.
- void init();
-
- /// Performs AND on the subtrees of operator.
- void and_filter_iterators();
-
- /// Performs OR on the subtrees of operator.
- void or_filter_iterators();
-
- /// Finds the next match for a filter on string field.
- void doc_matching_string_filter();
-
-public:
- uint32_t seq_id = 0;
- // Collection name -> references
- std::map<std::string, reference_filter_result_t> reference;
- Option<bool> status = Option(true);
-
- explicit filter_result_iterator_t(const std::string collection_name,
- Index const* const index, filter_node_t const* const filter_node) :
- collection_name(collection_name),
- index(index),
- filter_node(filter_node) {
- if (filter_node == nullptr) {
- is_valid = false;
- return;
- }
-
- // Generate the iterator tree and then initialize each node.
- if (filter_node->isOperator) {
- left_it = new filter_result_iterator_t(collection_name, index, filter_node->left);
- right_it = new filter_result_iterator_t(collection_name, index, filter_node->right);
- }
-
- init();
- }
-
- ~filter_result_iterator_t() {
- // In case the filter was on string field.
- for(auto expanded_plist: expanded_plists) {
- delete expanded_plist;
- }
+enum FILTER_OPERATOR {
+ AND,
+ OR
+};
- delete left_it;
- delete right_it;
+struct filter_node_t;
+struct field;
+
+struct filter {
+ std::string field_name;
+ std::vector<std::string> values;
+ std::vector<NUM_COMPARATOR> comparators;
+ // Would be set when `field: != ...` is encountered with a string field or `field: != [ ... ]` is encountered in the
+ // case of int and float fields. During filtering, all the results of matching the field against the values are
+ // aggregated and then this flag is checked if negation on the aggregated result is required.
+ bool apply_not_equals = false;
+
+ // Would store `Foo` in case of a filter expression like `$Foo(bar := baz)`
+ std::string referenced_collection_name = "";
+
+ static const std::string RANGE_OPERATOR() {
+ return "..";
}
- filter_result_iterator_t& operator=(filter_result_iterator_t&& obj) noexcept {
- if (&obj == this)
- return *this;
-
- // In case the filter was on string field.
- for(auto expanded_plist: expanded_plists) {
- delete expanded_plist;
- }
-
- delete left_it;
- delete right_it;
-
- collection_name = obj.collection_name;
- index = obj.index;
- filter_node = obj.filter_node;
- left_it = obj.left_it;
- right_it = obj.right_it;
-
- obj.left_it = nullptr;
- obj.right_it = nullptr;
+ static Option<bool> validate_numerical_filter_value(field _field, const std::string& raw_value);
- result_index = obj.result_index;
+ static Option<NUM_COMPARATOR> extract_num_comparator(std::string & comp_and_value);
- filter_result = std::move(obj.filter_result);
+ static Option<bool> parse_geopoint_filter_value(std::string& raw_value,
+ const std::string& format_err_msg,
+ std::string& processed_filter_val,
+ NUM_COMPARATOR& num_comparator);
- posting_list_iterators = std::move(obj.posting_list_iterators);
- expanded_plists = std::move(obj.expanded_plists);
-
- is_valid = obj.is_valid;
-
- seq_id = obj.seq_id;
- reference = std::move(obj.reference);
- status = std::move(obj.status);
+ static Option<bool> parse_filter_query(const std::string& filter_query,
+ const tsl::htrie_map<char, field>& search_schema,
+ const Store* store,
+ const std::string& doc_id_prefix,
+ filter_node_t*& root);
+};
- return *this;
+struct filter_node_t {
+ filter filter_exp;
+ FILTER_OPERATOR filter_operator;
+ bool isOperator;
+ filter_node_t* left = nullptr;
+ filter_node_t* right = nullptr;
+
+ filter_node_t(filter filter_exp)
+ : filter_exp(std::move(filter_exp)),
+ isOperator(false),
+ left(nullptr),
+ right(nullptr) {}
+
+ filter_node_t(FILTER_OPERATOR filter_operator,
+ filter_node_t* left,
+ filter_node_t* right)
+ : filter_operator(filter_operator),
+ isOperator(true),
+ left(left),
+ right(right) {}
+
+ ~filter_node_t() {
+ delete left;
+ delete right;
}
-
- /// Returns the status of the initialization of iterator tree.
- Option<bool> init_status();
-
- /// Returns true when doc and reference hold valid values. Used in conjunction with next() and skip_to(id).
- [[nodiscard]] bool valid();
-
- /// Returns a tri-state:
- /// 0: id is not valid
- /// 1: id is valid
- /// -1: end of iterator
- ///
- /// Handles moving the individual iterators internally.
- [[nodiscard]] int valid(uint32_t id);
-
- /// Advances the iterator to get the next value of doc and reference. The iterator may become invalid during this
- /// operation.
- void next();
-
- /// Advances the iterator until the doc value reaches or just overshoots id. The iterator may become invalid during
- /// this operation.
- void skip_to(uint32_t id);
-
- /// Returns true if at least one id from the posting list object matches the filter.
- bool contains_atleast_one(const void* obj);
-
- /// Returns to the initial state of the iterator.
- void reset();
-
- /// Iterates and collects all the filter ids into filter_array.
- /// \return size of the filter array
- uint32_t to_filter_id_array(uint32_t*& filter_array);
-
- uint32_t and_scalar(const uint32_t* A, const uint32_t& lenA, uint32_t*& results);
};
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.h
new file mode 100644
index 00000000..b67d67ca
--- /dev/null
+++ b/include/filter_result_iterator.h
@@ -0,0 +1,174 @@
+#pragma once
+
+#include <string>
+#include <map>
+#include <vector>
+#include "option.h"
+#include "posting_list.h"
+
+class Index;
+struct filter_node_t;
+
+struct reference_filter_result_t {
+ uint32_t count = 0;
+ uint32_t* docs = nullptr;
+
+ reference_filter_result_t& operator=(const reference_filter_result_t& obj) noexcept {
+ if (&obj == this)
+ return *this;
+
+ count = obj.count;
+ docs = new uint32_t[count];
+ memcpy(docs, obj.docs, count * sizeof(uint32_t));
+
+ return *this;
+ }
+
+ ~reference_filter_result_t() {
+ delete[] docs;
+ }
+};
+
+struct filter_result_t {
+ uint32_t count = 0;
+ uint32_t* docs = nullptr;
+ // Collection name -> Reference filter result
+ std::map<std::string, reference_filter_result_t*> reference_filter_results;
+
+ filter_result_t() = default;
+
+ filter_result_t(uint32_t count, uint32_t* docs) : count(count), docs(docs) {}
+
+ filter_result_t& operator=(const filter_result_t& obj) noexcept {
+ if (&obj == this)
+ return *this;
+
+ count = obj.count;
+ docs = new uint32_t[count];
+ memcpy(docs, obj.docs, count * sizeof(uint32_t));
+
+ // Copy every collection's references.
+ for (const auto &item: obj.reference_filter_results) {
+ reference_filter_results[item.first] = new reference_filter_result_t[count];
+
+ for (uint32_t i = 0; i < count; i++) {
+ reference_filter_results[item.first][i] = item.second[i];
+ }
+ }
+
+ return *this;
+ }
+
+ filter_result_t& operator=(filter_result_t&& obj) noexcept {
+ if (&obj == this)
+ return *this;
+
+ count = obj.count;
+ docs = obj.docs;
+ reference_filter_results = std::map(obj.reference_filter_results);
+
+ obj.docs = nullptr;
+ obj.reference_filter_results.clear();
+
+ return *this;
+ }
+
+ ~filter_result_t() {
+ delete[] docs;
+ for (const auto &item: reference_filter_results) {
+ delete[] item.second;
+ }
+ }
+
+ static void and_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result);
+
+ static void or_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result);
+};
+
+
+class filter_result_iterator_t {
+private:
+ std::string collection_name;
+ const Index* index = nullptr;
+ const filter_node_t* filter_node = nullptr;
+ filter_result_iterator_t* left_it = nullptr;
+ filter_result_iterator_t* right_it = nullptr;
+
+ /// Used in case of id and reference filter.
+ uint32_t result_index = 0;
+
+ /// Stores the result of the filters that cannot be iterated.
+ filter_result_t filter_result;
+
+ /// Initialized in case of filter on string field.
+ /// Sample filter values: ["foo bar", "baz"]. Each filter value is split into tokens. We get posting list iterator
+ /// for each token.
+ ///
+ /// Multiple filter values: Multiple tokens: posting list iterator
+ std::vector<std::vector<posting_list_t::iterator_t>> posting_list_iterators;
+ std::vector<posting_list_t*> expanded_plists;
+
+ /// Set to false when this iterator or it's subtree becomes invalid.
+ bool is_valid = true;
+
+ /// Initializes the state of iterator node after it's creation.
+ void init();
+
+ /// Performs AND on the subtrees of operator.
+ void and_filter_iterators();
+
+ /// Performs OR on the subtrees of operator.
+ void or_filter_iterators();
+
+ /// Finds the next match for a filter on string field.
+ void doc_matching_string_filter(bool field_is_array);
+
+public:
+ uint32_t seq_id = 0;
+ // Collection name -> references
+ std::map<std::string, reference_filter_result_t> reference;
+ Option<bool> status = Option(true);
+
+ explicit filter_result_iterator_t(const std::string collection_name,
+ Index const* const index, filter_node_t const* const filter_node);
+
+ ~filter_result_iterator_t();
+
+ filter_result_iterator_t& operator=(filter_result_iterator_t&& obj) noexcept;
+
+ /// Returns the status of the initialization of iterator tree.
+ Option<bool> init_status();
+
+ /// Returns true when doc and reference hold valid values. Used in conjunction with next() and skip_to(id).
+ [[nodiscard]] bool valid();
+
+ /// Returns a tri-state:
+ /// 0: id is not valid
+ /// 1: id is valid
+ /// -1: end of iterator
+ ///
+ /// Handles moving the individual iterators internally.
+ [[nodiscard]] int valid(uint32_t id);
+
+ /// Advances the iterator to get the next value of doc and reference. The iterator may become invalid during this
+ /// operation.
+ void next();
+
+ /// Advances the iterator until the doc value reaches or just overshoots id. The iterator may become invalid during
+ /// this operation.
+ void skip_to(uint32_t id);
+
+ /// Returns true if at least one id from the posting list object matches the filter.
+ bool contains_atleast_one(const void* obj);
+
+ /// Returns to the initial state of the iterator.
+ void reset();
+
+ /// Iterates and collects all the filter ids into filter_array.
+ /// \return size of the filter array
+ uint32_t to_filter_id_array(uint32_t*& filter_array);
+
+ /// Performs AND with the contents of A and allocates a new array of results.
+ /// \return size of the results array
+ uint32_t and_scalar(const uint32_t* A, const uint32_t& lenA, uint32_t*& results);
+};
diff --git a/include/index.h b/include/index.h
index 63681b9e..c65b84c5 100644
--- a/include/index.h
+++ b/include/index.h
@@ -13,7 +13,6 @@
#include <topster.h>
#include <json.hpp>
#include <field.h>
-#include <validator.h>
#include <option.h>
#include <set>
#include "string_utils.h"
@@ -30,6 +29,7 @@
#include "override.h"
#include "vector_query_ops.h"
#include "hnswlib/hnswlib.h"
+#include "filter.h"
static constexpr size_t ARRAY_FACET_DIM = 4;
using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;
@@ -233,19 +233,15 @@ struct index_record {
};
class VectorFilterFunctor: public hnswlib::BaseFilterFunctor {
- const uint32_t* filter_ids = nullptr;
- const uint32_t filter_ids_length = 0;
+ filter_result_iterator_t* const filter_result_iterator;
public:
- explicit VectorFilterFunctor(const uint32_t* filter_ids, const uint32_t filter_ids_length) :
- filter_ids(filter_ids), filter_ids_length(filter_ids_length) {}
+ explicit VectorFilterFunctor(filter_result_iterator_t* const filter_result_iterator) :
+ filter_result_iterator(filter_result_iterator) {}
- bool operator()(hnswlib::labeltype id) override {
- if(filter_ids_length == 0) {
- return true;
- }
-
- return std::binary_search(filter_ids, filter_ids + filter_ids_length, id);
+ bool operator()(unsigned int id) {
+ filter_result_iterator->reset();
+ return filter_result_iterator->valid(id) == 1;
}
};
@@ -412,7 +408,7 @@ private:
void search_all_candidates(const size_t num_search_fields,
const text_match_type_t match_type,
const std::vector<search_field_t>& the_fields,
- const uint32_t* filter_ids, size_t filter_ids_length,
+ filter_result_iterator_t& filter_result_iterator,
const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
const std::unordered_set<uint32_t>& excluded_group_ids,
const std::vector<sort_by>& sort_fields,
@@ -563,6 +559,9 @@ public:
// in the query that have the least individual hits one by one until enough results are found.
static const int DROP_TOKENS_THRESHOLD = 1;
+ // "_all_" is a special field that maps to all the ids in the index.
+ static constexpr const char* SEQ_IDS_FILTER = "_all_: 1";
+
Index() = delete;
Index(const std::string& name,
@@ -741,8 +740,9 @@ public:
const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,
const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,
- uint32_t*& all_result_ids, size_t& all_result_ids_len, const uint32_t* filter_ids,
- uint32_t filter_ids_length, const size_t concurrency,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ filter_result_iterator_t& filter_result_iterator, const uint32_t& approx_filter_ids_length,
+ const size_t concurrency,
const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
const std::vector<size_t>& geopoint_indices) const;
@@ -782,7 +782,7 @@ public:
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,
const size_t max_extra_suffix, const std::vector<token_t>& query_tokens, Topster* actual_topster,
- const uint32_t *filter_ids, size_t filter_ids_length,
+ filter_result_iterator_t& filter_result_iterator,
const int sort_order[3],
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
const std::vector<size_t>& geopoint_indices,
@@ -813,7 +813,7 @@ public:
spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries,
uint32_t*& all_result_ids, size_t& all_result_ids_len,
- const uint32_t* filter_ids, uint32_t filter_ids_length,
+ filter_result_iterator_t& filter_result_iterator,
std::set<uint64>& query_hashes,
const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
@@ -838,7 +838,7 @@ public:
Topster* curated_topster,
const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
bool is_wildcard_query,
- uint32_t*& filter_ids, uint32_t& filter_ids_length) const;
+ uint32_t*& phrase_result_ids, uint32_t& phrase_result_count) const;
void fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
const std::vector<token_t>& query_tokens,
@@ -846,7 +846,7 @@ public:
const text_match_type_t match_type,
const uint32_t* exclude_token_ids,
size_t exclude_token_ids_size,
- const uint32_t* filter_ids, size_t filter_ids_length,
+ filter_result_iterator_t& filter_result_iterator,
const std::vector<uint32_t>& curated_ids,
const std::unordered_set<uint32_t>& excluded_group_ids,
const std::vector<sort_by>& sort_fields,
@@ -875,7 +875,7 @@ public:
const std::string& previous_token_str,
const std::vector<search_field_t>& the_fields,
const size_t num_search_fields,
- const uint32_t* filter_ids, uint32_t filter_ids_length,
+ filter_result_iterator_t& filter_result_iterator,
const uint32_t* exclude_token_ids,
size_t exclude_token_ids_size,
std::vector<uint32_t>& prev_token_doc_ids,
@@ -897,7 +897,7 @@ public:
const std::vector<std::string>& group_by_fields,
bool prioritize_exact_match,
const bool search_all_candidates,
- const uint32_t* filter_ids, uint32_t filter_ids_length,
+ filter_result_iterator_t& filter_result_iterator,
const uint32_t total_cost,
const int syn_orig_num_tokens,
const uint32_t* exclude_token_ids,
@@ -946,14 +946,14 @@ public:
int64_t* scores,
int64_t& match_score_index) const;
- void
- process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
- const std::vector<uint32_t>& excluded_ids, const std::vector<std::string>& group_by_fields,
- const size_t group_limit, const bool filter_curated_hits, const uint32_t* filter_ids,
- uint32_t filter_ids_length, std::set<uint32_t>& curated_ids,
- std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- std::vector<uint32_t>& included_ids_vec,
- std::unordered_set<uint32_t>& excluded_group_ids) const;
+ void process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
+ const std::vector<uint32_t>& excluded_ids, const std::vector<std::string>& group_by_fields,
+ const size_t group_limit, const bool filter_curated_hits,
+ filter_result_iterator_t& filter_result_iterator,
+ std::set<uint32_t>& curated_ids,
+ std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ std::vector<uint32_t>& included_ids_vec,
+ std::unordered_set<uint32_t>& excluded_group_ids) const;
Option<bool> seq_ids_outside_top_k(const std::string& field_name, size_t k,
std::vector<uint32_t>& outside_seq_ids);
diff --git a/include/posting_list.h b/include/posting_list.h
index 16f42ea7..11ed9c91 100644
--- a/include/posting_list.h
+++ b/include/posting_list.h
@@ -8,6 +8,7 @@
#include "thread_local_vars.h"
typedef uint32_t last_id_t;
+class filter_result_iterator_t;
struct result_iter_state_t {
const uint32_t* excluded_result_ids = nullptr;
@@ -19,12 +20,19 @@ struct result_iter_state_t {
size_t excluded_result_ids_index = 0;
size_t filter_ids_index = 0;
+ filter_result_iterator_t* fit = nullptr;
+
result_iter_state_t() = default;
result_iter_state_t(const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,
const uint32_t* filter_ids, const size_t filter_ids_length) : excluded_result_ids(excluded_result_ids),
excluded_result_ids_size(excluded_result_ids_size),
filter_ids(filter_ids), filter_ids_length(filter_ids_length) {}
+
+ result_iter_state_t(const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,
+ filter_result_iterator_t* fit) : excluded_result_ids(excluded_result_ids),
+ excluded_result_ids_size(excluded_result_ids_size),
+ fit(fit){}
};
/*
@@ -186,6 +194,9 @@ public:
const uint32_t* ids, const uint32_t num_ids,
uint32_t*& exact_ids, size_t& num_exact_ids);
+ static bool has_exact_match(std::vector<posting_list_t::iterator_t>& posting_list_iterators,
+ const bool field_is_array);
+
static void get_phrase_matches(std::vector<iterator_t>& its, bool field_is_array,
const uint32_t* ids, const uint32_t num_ids,
uint32_t*& phrase_ids, size_t& num_phrase_ids);
diff --git a/include/topster.h b/include/topster.h
index e59ae74c..8e10abcb 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -6,6 +6,7 @@
#include <algorithm>
#include <unordered_map>
#include <field.h>
+#include "filter_result_iterator.h"
struct KV {
int8_t match_score_index{};
diff --git a/include/validator.h b/include/validator.h
index 3998f5dc..a8a4a8f9 100644
--- a/include/validator.h
+++ b/include/validator.h
@@ -6,21 +6,6 @@
#include "tsl/htrie_map.h"
#include "field.h"
-enum index_operation_t {
- CREATE,
- UPSERT,
- UPDATE,
- EMPLACE,
- DELETE
-};
-
-enum class DIRTY_VALUES {
- REJECT = 1,
- DROP = 2,
- COERCE_OR_REJECT = 3,
- COERCE_OR_DROP = 4,
-};
-
class validator_t {
public:
diff --git a/src/art.cpp b/src/art.cpp
index 40b028a3..48470551 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -22,6 +22,7 @@
#include "art.h"
#include "logger.h"
#include "array_utils.h"
+#include "filter_result_iterator.h"
/**
* Macros to manipulate pointer tags
@@ -972,6 +973,36 @@ const uint32_t* get_allowed_doc_ids(art_tree *t, const std::string& prev_token,
return prev_token_doc_ids;
}
+const uint32_t* get_allowed_doc_ids(art_tree *t, const std::string& prev_token,
+ filter_result_iterator_t& filter_result_iterator,
+ size_t& prev_token_doc_ids_len) {
+
+ art_leaf* prev_leaf = static_cast<art_leaf*>(
+ art_search(t, reinterpret_cast<const unsigned char*>(prev_token.c_str()), prev_token.size() + 1)
+ );
+
+ uint32_t* prev_token_doc_ids = nullptr;
+
+ if(prev_token.empty() || !prev_leaf) {
+ prev_token_doc_ids_len = filter_result_iterator.to_filter_id_array(prev_token_doc_ids);
+ return prev_token_doc_ids;
+ }
+
+ std::vector<uint32_t> prev_leaf_ids;
+ posting_t::merge({prev_leaf->values}, prev_leaf_ids);
+
+ if(filter_result_iterator.valid()) {
+ prev_token_doc_ids_len = filter_result_iterator.and_scalar(prev_leaf_ids.data(), prev_leaf_ids.size(),
+ prev_token_doc_ids);
+ } else {
+ prev_token_doc_ids_len = prev_leaf_ids.size();
+ prev_token_doc_ids = new uint32_t[prev_token_doc_ids_len];
+ std::copy(prev_leaf_ids.begin(), prev_leaf_ids.end(), prev_token_doc_ids);
+ }
+
+ return prev_token_doc_ids;
+}
+
bool validate_and_add_leaf(art_leaf* leaf, const bool last_token, const std::string& prev_token,
const uint32_t* allowed_doc_ids, const size_t allowed_doc_ids_len,
std::set<std::string>& exclude_leaves, const art_leaf* exact_leaf,
@@ -1622,6 +1653,165 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
return 0;
}
+int art_fuzzy_search_i(art_tree *t, const unsigned char *term, const int term_len, const int min_cost, const int max_cost,
+ const size_t max_words, const token_ordering token_order,
+ const bool prefix, bool last_token, const std::string& prev_token,
+ filter_result_iterator_t& filter_result_iterator,
+ std::vector<art_leaf *> &results, std::set<std::string>& exclude_leaves) {
+
+ std::vector<const art_node*> nodes;
+ int irow[term_len + 1];
+ int jrow[term_len + 1];
+ for (int i = 0; i <= term_len; i++){
+ irow[i] = jrow[i] = i;
+ }
+
+ //auto begin = std::chrono::high_resolution_clock::now();
+
+ if(IS_LEAF(t->root)) {
+ art_leaf *l = (art_leaf *) LEAF_RAW(t->root);
+ art_fuzzy_recurse(0, l->key[0], t->root, 0, term, term_len, irow, jrow, min_cost, max_cost, prefix, nodes);
+ } else {
+ if(t->root == nullptr) {
+ return 0;
+ }
+
+ // send depth as -1 to indicate that this is a root node
+ art_fuzzy_recurse(0, 0, t->root, -1, term, term_len, irow, jrow, min_cost, max_cost, prefix, nodes);
+ }
+
+ //long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
+ //!LOG(INFO) << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size();
+
+ //auto begin = std::chrono::high_resolution_clock::now();
+
+ size_t key_len = prefix ? term_len + 1 : term_len;
+ art_leaf* exact_leaf = (art_leaf *) art_search(t, term, key_len);
+ //LOG(INFO) << "exact_leaf: " << exact_leaf << ", term: " << term << ", term_len: " << term_len;
+
+ // documents that contain the previous token and/or filter ids
+ size_t allowed_doc_ids_len = 0;
+ const uint32_t* allowed_doc_ids = get_allowed_doc_ids(t, prev_token, filter_result_iterator, allowed_doc_ids_len);
+
+ for(auto node: nodes) {
+ art_topk_iter(node, token_order, max_words, exact_leaf,
+ last_token, prev_token, allowed_doc_ids, allowed_doc_ids_len,
+ t, exclude_leaves, results);
+ }
+
+ if(token_order == FREQUENCY) {
+ std::sort(results.begin(), results.end(), compare_art_leaf_frequency);
+ } else {
+ std::sort(results.begin(), results.end(), compare_art_leaf_score);
+ }
+
+ if(exact_leaf && min_cost == 0) {
+ std::string tok(reinterpret_cast<char*>(exact_leaf->key), exact_leaf->key_len - 1);
+ if(exclude_leaves.count(tok) == 0) {
+ results.insert(results.begin(), exact_leaf);
+ exclude_leaves.emplace(tok);
+ }
+ }
+
+ if(results.size() > max_words) {
+ results.resize(max_words);
+ }
+
+ /*auto time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
+ if(time_micro > 1000) {
+ LOG(INFO) << "Time taken for art_topk_iter: " << time_micro
+ << "us, size of nodes: " << nodes.size()
+ << ", filter_ids_length: " << filter_ids_length;
+ }*/
+
+// TODO: Figure out this edge case.
+// if(allowed_doc_ids != filter_ids) {
+// delete [] allowed_doc_ids;
+// }
+
+ return 0;
+}
+
+int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int min_cost, const int max_cost,
+ const size_t max_words, const token_ordering token_order, const bool prefix,
+ bool last_token, const std::string& prev_token,
+ filter_result_iterator_t& filter_result_iterator,
+ std::vector<art_leaf *> &results, std::set<std::string>& exclude_leaves) {
+
+ std::vector<const art_node*> nodes;
+ int irow[term_len + 1];
+ int jrow[term_len + 1];
+ for (int i = 0; i <= term_len; i++){
+ irow[i] = jrow[i] = i;
+ }
+
+ //auto begin = std::chrono::high_resolution_clock::now();
+
+ if(IS_LEAF(t->root)) {
+ art_leaf *l = (art_leaf *) LEAF_RAW(t->root);
+ art_fuzzy_recurse(0, l->key[0], t->root, 0, term, term_len, irow, jrow, min_cost, max_cost, prefix, nodes);
+ } else {
+ if(t->root == nullptr) {
+ return 0;
+ }
+
+ // send depth as -1 to indicate that this is a root node
+ art_fuzzy_recurse(0, 0, t->root, -1, term, term_len, irow, jrow, min_cost, max_cost, prefix, nodes);
+ }
+
+ //long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
+ //!LOG(INFO) << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size();
+
+ //auto begin = std::chrono::high_resolution_clock::now();
+
+ size_t key_len = prefix ? term_len + 1 : term_len;
+ art_leaf* exact_leaf = (art_leaf *) art_search(t, term, key_len);
+ //LOG(INFO) << "exact_leaf: " << exact_leaf << ", term: " << term << ", term_len: " << term_len;
+
+ // documents that contain the previous token and/or filter ids
+ size_t allowed_doc_ids_len = 0;
+ const uint32_t* allowed_doc_ids = get_allowed_doc_ids(t, prev_token, filter_result_iterator, allowed_doc_ids_len);
+
+ for(auto node: nodes) {
+ art_topk_iter(node, token_order, max_words, exact_leaf,
+ last_token, prev_token, allowed_doc_ids, allowed_doc_ids_len,
+ t, exclude_leaves, results);
+ }
+
+ if(token_order == FREQUENCY) {
+ std::sort(results.begin(), results.end(), compare_art_leaf_frequency);
+ } else {
+ std::sort(results.begin(), results.end(), compare_art_leaf_score);
+ }
+
+ if(exact_leaf && min_cost == 0) {
+ std::string tok(reinterpret_cast<char*>(exact_leaf->key), exact_leaf->key_len - 1);
+ if(exclude_leaves.count(tok) == 0) {
+ results.insert(results.begin(), exact_leaf);
+ exclude_leaves.emplace(tok);
+ }
+ }
+
+ if(results.size() > max_words) {
+ results.resize(max_words);
+ }
+
+ /*auto time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
+
+ if(time_micro > 1000) {
+ LOG(INFO) << "Time taken for art_topk_iter: " << time_micro
+ << "us, size of nodes: " << nodes.size()
+ << ", filter_ids_length: " << filter_ids_length;
+ }*/
+
+// TODO: Figure out this edge case.
+// if(allowed_doc_ids != filter_ids) {
+// delete [] allowed_doc_ids;
+// }
+
+ return 0;
+}
+
void encode_int32(int32_t n, unsigned char *chars) {
unsigned char symbols[16] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
diff --git a/src/collection.cpp b/src/collection.cpp
index 63db8c19..21f2d187 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -12,6 +12,7 @@
#include <regex>
#include <list>
#include <posting.h>
+#include "validator.h"
#include "topster.h"
#include "logger.h"
#include "thread_local_vars.h"
diff --git a/src/field.cpp b/src/field.cpp
index ee62b7fa..f48ecf50 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -6,511 +6,6 @@
#include <collection_manager.h>
#include <regex>
-Option<bool> filter::parse_geopoint_filter_value(std::string& raw_value,
- const std::string& format_err_msg,
- std::string& processed_filter_val,
- NUM_COMPARATOR& num_comparator) {
-
- num_comparator = LESS_THAN_EQUALS;
-
- if(!(raw_value[0] == '(' && raw_value[raw_value.size() - 1] == ')')) {
- return Option<bool>(400, format_err_msg);
- }
-
- std::vector<std::string> filter_values;
- auto raw_val_without_paran = raw_value.substr(1, raw_value.size() - 2);
- StringUtils::split(raw_val_without_paran, filter_values, ",");
-
- // we will end up with: "10.45 34.56 2 km" or "10.45 34.56 2mi" or a geo polygon
-
- if(filter_values.size() < 3) {
- return Option<bool>(400, format_err_msg);
- }
-
- // do validation: format should match either a point + radius or polygon
-
- size_t num_floats = 0;
- for(const auto& fvalue: filter_values) {
- if(StringUtils::is_float(fvalue)) {
- num_floats++;
- }
- }
-
- bool is_polygon = (num_floats == filter_values.size());
- if(!is_polygon) {
- // we have to ensure that this is a point + radius match
- if(!StringUtils::is_float(filter_values[0]) || !StringUtils::is_float(filter_values[1])) {
- return Option<bool>(400, format_err_msg);
- }
-
- if(filter_values[0] == "nan" || filter_values[0] == "NaN" ||
- filter_values[1] == "nan" || filter_values[1] == "NaN") {
- return Option<bool>(400, format_err_msg);
- }
- }
-
- if(is_polygon) {
- processed_filter_val = raw_val_without_paran;
- } else {
- // point + radius
- // filter_values[2] is distance, get the unit, validate it and split on that
- if(filter_values[2].size() < 2) {
- return Option<bool>(400, "Unit must be either `km` or `mi`.");
- }
-
- std::string unit = filter_values[2].substr(filter_values[2].size()-2, 2);
-
- if(unit != "km" && unit != "mi") {
- return Option<bool>(400, "Unit must be either `km` or `mi`.");
- }
-
- std::vector<std::string> dist_values;
- StringUtils::split(filter_values[2], dist_values, unit);
-
- if(dist_values.size() != 1) {
- return Option<bool>(400, format_err_msg);
- }
-
- if(!StringUtils::is_float(dist_values[0])) {
- return Option<bool>(400, format_err_msg);
- }
-
- processed_filter_val = filter_values[0] + ", " + filter_values[1] + ", " + // co-ords
- dist_values[0] + ", " + unit; // X km
- }
-
- return Option<bool>(true);
-}
-
-bool isOperator(const std::string& expression) {
- return expression == "&&" || expression == "||";
-}
-
-// https://en.wikipedia.org/wiki/Shunting_yard_algorithm
-Option<bool> toPostfix(std::queue<std::string>& tokens, std::queue<std::string>& postfix) {
- std::stack<std::string> operatorStack;
-
- while (!tokens.empty()) {
- auto expression = tokens.front();
- tokens.pop();
-
- if (isOperator(expression)) {
- // We only have two operators &&, || having the same precedence and both being left associative.
- while (!operatorStack.empty() && operatorStack.top() != "(") {
- postfix.push(operatorStack.top());
- operatorStack.pop();
- }
-
- operatorStack.push(expression);
- } else if (expression == "(") {
- operatorStack.push(expression);
- } else if (expression == ")") {
- while (!operatorStack.empty() && operatorStack.top() != "(") {
- postfix.push(operatorStack.top());
- operatorStack.pop();
- }
-
- if (operatorStack.empty() || operatorStack.top() != "(") {
- return Option<bool>(400, "Could not parse the filter query: unbalanced parentheses.");
- }
- operatorStack.pop();
- } else {
- postfix.push(expression);
- }
- }
-
- while (!operatorStack.empty()) {
- if (operatorStack.top() == "(") {
- return Option<bool>(400, "Could not parse the filter query: unbalanced parentheses.");
- }
- postfix.push(operatorStack.top());
- operatorStack.pop();
- }
-
- return Option<bool>(true);
-}
-
-Option<bool> toMultiValueNumericFilter(std::string& raw_value, filter& filter_exp, const field& _field) {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
- filter_exp = {_field.name, {}, {}};
- for (std::string& filter_value: filter_values) {
- Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(filter_value);
- if (!op_comparator.ok()) {
- return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());
- }
- if (op_comparator.get() == RANGE_INCLUSIVE) {
- // split the value around range operator to extract bounds
- std::vector<std::string> range_values;
- StringUtils::split(filter_value, range_values, filter::RANGE_OPERATOR());
- for (const std::string& range_value: range_values) {
- auto validate_op = filter::validate_numerical_filter_value(_field, range_value);
- if (!validate_op.ok()) {
- return validate_op;
- }
- filter_exp.values.push_back(range_value);
- filter_exp.comparators.push_back(op_comparator.get());
- }
- } else {
- auto validate_op = filter::validate_numerical_filter_value(_field, filter_value);
- if (!validate_op.ok()) {
- return validate_op;
- }
- filter_exp.values.push_back(filter_value);
- filter_exp.comparators.push_back(op_comparator.get());
- }
- }
-
- return Option<bool>(true);
-}
-
-Option<bool> toFilter(const std::string expression,
- filter& filter_exp,
- const tsl::htrie_map<char, field>& search_schema,
- const Store* store,
- const std::string& doc_id_prefix) {
- // split into [field_name, value]
- size_t found_index = expression.find(':');
- if (found_index == std::string::npos) {
- return Option<bool>(400, "Could not parse the filter query.");
- }
- std::string&& field_name = expression.substr(0, found_index);
- StringUtils::trim(field_name);
- if (field_name == "id") {
- std::string&& raw_value = expression.substr(found_index + 1, std::string::npos);
- StringUtils::trim(raw_value);
- std::string empty_filter_err = "Error with filter field `id`: Filter value cannot be empty.";
- if (raw_value.empty()) {
- return Option<bool>(400, empty_filter_err);
- }
- filter_exp = {field_name, {}, {}};
- NUM_COMPARATOR id_comparator = EQUALS;
- size_t filter_value_index = 0;
- if (raw_value[0] == '=') {
- id_comparator = EQUALS;
- while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
- } else if (raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
- return Option<bool>(400, "Not equals filtering is not supported on the `id` field.");
- }
- if (filter_value_index != 0) {
- raw_value = raw_value.substr(filter_value_index);
- }
- if (raw_value.empty()) {
- return Option<bool>(400, empty_filter_err);
- }
- if (raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> doc_ids;
- StringUtils::split_to_values(raw_value.substr(1, raw_value.size() - 2), doc_ids);
- for (std::string& doc_id: doc_ids) {
- // we have to convert the doc_id to seq id
- std::string seq_id_str;
- StoreStatus seq_id_status = store->get(doc_id_prefix + doc_id, seq_id_str);
- if (seq_id_status != StoreStatus::FOUND) {
- continue;
- }
- filter_exp.values.push_back(seq_id_str);
- filter_exp.comparators.push_back(id_comparator);
- }
- } else {
- std::vector<std::string> doc_ids;
- StringUtils::split_to_values(raw_value, doc_ids); // to handle backticks
- std::string seq_id_str;
- StoreStatus seq_id_status = store->get(doc_id_prefix + doc_ids[0], seq_id_str);
- if (seq_id_status == StoreStatus::FOUND) {
- filter_exp.values.push_back(seq_id_str);
- filter_exp.comparators.push_back(id_comparator);
- }
- }
- return Option<bool>(true);
- }
-
- auto field_it = search_schema.find(field_name);
-
- if (field_it == search_schema.end()) {
- return Option<bool>(404, "Could not find a filter field named `" + field_name + "` in the schema.");
- }
-
- if (field_it->num_dim > 0) {
- return Option<bool>(404, "Cannot filter on vector field `" + field_name + "`.");
- }
-
- const field& _field = field_it.value();
- std::string&& raw_value = expression.substr(found_index + 1, std::string::npos);
- StringUtils::trim(raw_value);
- // skip past optional `:=` operator, which has no meaning for non-string fields
- if (!_field.is_string() && raw_value[0] == '=') {
- size_t filter_value_index = 0;
- while (raw_value[++filter_value_index] == ' ');
- raw_value = raw_value.substr(filter_value_index);
- }
- if (_field.is_integer() || _field.is_float()) {
- // could be a single value or a list
- if (raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- Option<bool> op = toMultiValueNumericFilter(raw_value, filter_exp, _field);
- if (!op.ok()) {
- return op;
- }
- } else {
- Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);
- if (!op_comparator.ok()) {
- return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());
- }
- if (op_comparator.get() == RANGE_INCLUSIVE) {
- // split the value around range operator to extract bounds
- std::vector<std::string> range_values;
- StringUtils::split(raw_value, range_values, filter::RANGE_OPERATOR());
- filter_exp.field_name = field_name;
- for (const std::string& range_value: range_values) {
- auto validate_op = filter::validate_numerical_filter_value(_field, range_value);
- if (!validate_op.ok()) {
- return validate_op;
- }
- filter_exp.values.push_back(range_value);
- filter_exp.comparators.push_back(op_comparator.get());
- }
- } else if (op_comparator.get() == NOT_EQUALS && raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- Option<bool> op = toMultiValueNumericFilter(raw_value, filter_exp, _field);
- if (!op.ok()) {
- return op;
- }
- filter_exp.apply_not_equals = true;
- } else {
- auto validate_op = filter::validate_numerical_filter_value(_field, raw_value);
- if (!validate_op.ok()) {
- return validate_op;
- }
- filter_exp = {field_name, {raw_value}, {op_comparator.get()}};
- }
- }
- } else if (_field.is_bool()) {
- NUM_COMPARATOR bool_comparator = EQUALS;
- size_t filter_value_index = 0;
- if (raw_value[0] == '=') {
- bool_comparator = EQUALS;
- while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
- } else if (raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
- bool_comparator = NOT_EQUALS;
- filter_value_index++;
- while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
- }
- if (filter_value_index != 0) {
- raw_value = raw_value.substr(filter_value_index);
- }
- if (filter_value_index == raw_value.size()) {
- return Option<bool>(400, "Error with filter field `" + _field.name +
- "`: Filter value cannot be empty.");
- }
- if (raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
- filter_exp = {field_name, {}, {}};
- for (std::string& filter_value: filter_values) {
- if (filter_value != "true" && filter_value != "false") {
- return Option<bool>(400, "Values of filter field `" + _field.name +
- "`: must be `true` or `false`.");
- }
- filter_value = (filter_value == "true") ? "1" : "0";
- filter_exp.values.push_back(filter_value);
- filter_exp.comparators.push_back(bool_comparator);
- }
- } else {
- if (raw_value != "true" && raw_value != "false") {
- return Option<bool>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");
- }
- std::string bool_value = (raw_value == "true") ? "1" : "0";
- filter_exp = {field_name, {bool_value}, {bool_comparator}};
- }
- } else if (_field.is_geopoint()) {
- filter_exp = {field_name, {}, {}};
- const std::string& format_err_msg = "Value of filter field `" + _field.name +
- "`: must be in the `(-44.50, 170.29, 0.75 km)` or "
- "(56.33, -65.97, 23.82, -127.82) format.";
- NUM_COMPARATOR num_comparator;
- // could be a single value or a list
- if (raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, "),");
- for (std::string& filter_value: filter_values) {
- filter_value += ")";
- std::string processed_filter_val;
- auto parse_op = filter::parse_geopoint_filter_value(filter_value, format_err_msg, processed_filter_val,
- num_comparator);
- if (!parse_op.ok()) {
- return parse_op;
- }
- filter_exp.values.push_back(processed_filter_val);
- filter_exp.comparators.push_back(num_comparator);
- }
- } else {
- // single value, e.g. (10.45, 34.56, 2 km)
- std::string processed_filter_val;
- auto parse_op = filter::parse_geopoint_filter_value(raw_value, format_err_msg, processed_filter_val,
- num_comparator);
- if (!parse_op.ok()) {
- return parse_op;
- }
- filter_exp.values.push_back(processed_filter_val);
- filter_exp.comparators.push_back(num_comparator);
- }
- } else if (_field.is_string()) {
- size_t filter_value_index = 0;
- NUM_COMPARATOR str_comparator = CONTAINS;
- if (raw_value[0] == '=') {
- // string filter should be evaluated in strict "equals" mode
- str_comparator = EQUALS;
- while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
- } else if (raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
- str_comparator = NOT_EQUALS;
- filter_value_index++;
- while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
- }
- if (filter_value_index == raw_value.size()) {
- return Option<bool>(400, "Error with filter field `" + _field.name +
- "`: Filter value cannot be empty.");
- }
- if (raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split_to_values(
- raw_value.substr(filter_value_index + 1, raw_value.size() - filter_value_index - 2), filter_values);
- filter_exp = {field_name, filter_values, {str_comparator}};
- } else {
- filter_exp = {field_name, {raw_value.substr(filter_value_index)}, {str_comparator}};
- }
-
- filter_exp.apply_not_equals = (str_comparator == NOT_EQUALS);
- } else {
- return Option<bool>(400, "Error with filter field `" + _field.name +
- "`: Unidentified field data type, see docs for supported data types.");
- }
-
- return Option<bool>(true);
-}
-
-// https://stackoverflow.com/a/423914/11218270
-Option<bool> toParseTree(std::queue<std::string>& postfix, filter_node_t*& root,
- const tsl::htrie_map<char, field>& search_schema,
- const Store* store,
- const std::string& doc_id_prefix) {
- std::stack<filter_node_t*> nodeStack;
- bool is_successful = true;
- std::string error_message;
-
- filter_node_t *filter_node = nullptr;
-
- while (!postfix.empty()) {
- const std::string expression = postfix.front();
- postfix.pop();
-
- if (isOperator(expression)) {
- if (nodeStack.empty()) {
- is_successful = false;
- error_message = "Could not parse the filter query: unbalanced `" + expression + "` operands.";
- break;
- }
- auto operandB = nodeStack.top();
- nodeStack.pop();
-
- if (nodeStack.empty()) {
- delete operandB;
- is_successful = false;
- error_message = "Could not parse the filter query: unbalanced `" + expression + "` operands.";
- break;
- }
- auto operandA = nodeStack.top();
- nodeStack.pop();
-
- filter_node = new filter_node_t(expression == "&&" ? AND : OR, operandA, operandB);
- } else {
- filter filter_exp;
-
- // Expected value: $Collection(...)
- bool is_referenced_filter = (expression[0] == '$' && expression[expression.size() - 1] == ')');
- if (is_referenced_filter) {
- size_t parenthesis_index = expression.find('(');
-
- std::string collection_name = expression.substr(1, parenthesis_index - 1);
- auto &cm = CollectionManager::get_instance();
- auto collection = cm.get_collection(collection_name);
- if (collection == nullptr) {
- is_successful = false;
- error_message = "Referenced collection `" + collection_name + "` not found.";
- break;
- }
-
- filter_exp = {expression.substr(parenthesis_index + 1, expression.size() - parenthesis_index - 2)};
- filter_exp.referenced_collection_name = collection_name;
- } else {
- Option<bool> toFilter_op = toFilter(expression, filter_exp, search_schema, store, doc_id_prefix);
- if (!toFilter_op.ok()) {
- is_successful = false;
- error_message = toFilter_op.error();
- break;
- }
- }
-
- filter_node = new filter_node_t(filter_exp);
- }
-
- nodeStack.push(filter_node);
- }
-
- if (!is_successful) {
- while (!nodeStack.empty()) {
- auto filterNode = nodeStack.top();
- delete filterNode;
- nodeStack.pop();
- }
-
- return Option<bool>(400, error_message);
- }
-
- if (nodeStack.empty()) {
- return Option<bool>(400, "Filter query cannot be empty.");
- }
- root = nodeStack.top();
-
- return Option<bool>(true);
-}
-
-Option<bool> filter::parse_filter_query(const std::string& filter_query,
- const tsl::htrie_map<char, field>& search_schema,
- const Store* store,
- const std::string& doc_id_prefix,
- filter_node_t*& root) {
- auto _filter_query = filter_query;
- StringUtils::trim(_filter_query);
- if (_filter_query.empty()) {
- return Option<bool>(true);
- }
-
- std::queue<std::string> tokens;
- Option<bool> tokenize_op = StringUtils::tokenize_filter_query(filter_query, tokens);
- if (!tokenize_op.ok()) {
- return tokenize_op;
- }
-
- if (tokens.size() > 100) {
- return Option<bool>(400, "Filter expression is not valid.");
- }
-
- std::queue<std::string> postfix;
- Option<bool> toPostfix_op = toPostfix(tokens, postfix);
- if (!toPostfix_op.ok()) {
- return toPostfix_op;
- }
-
- Option<bool> toParseTree_op = toParseTree(postfix,
- root,
- search_schema,
- store,
- doc_id_prefix);
- if (!toParseTree_op.ok()) {
- return toParseTree_op;
- }
-
- return Option<bool>(true);
-}
-
Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::json& field_json,
std::vector<field>& the_fields,
string& fallback_field_type, size_t& num_auto_detect_fields) {
@@ -1046,167 +541,3 @@ void field::compact_nested_fields(tsl::htrie_map<char, field>& nested_fields) {
nested_fields.erase_prefix(field_name + ".");
}
}
-
-void filter_result_t::and_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {
- auto lenA = a.count, lenB = b.count;
- if (lenA == 0 || lenB == 0) {
- return;
- }
-
- result.docs = new uint32_t[std::min(lenA, lenB)];
-
- auto A = a.docs, B = b.docs, out = result.docs;
- const uint32_t *endA = A + lenA;
- const uint32_t *endB = B + lenB;
-
- // Add an entry of references in the result for each unique collection in a and b.
- for (auto const& item: a.reference_filter_results) {
- if (result.reference_filter_results.count(item.first) == 0) {
- result.reference_filter_results[item.first] = new reference_filter_result_t[std::min(lenA, lenB)];
- }
- }
- for (auto const& item: b.reference_filter_results) {
- if (result.reference_filter_results.count(item.first) == 0) {
- result.reference_filter_results[item.first] = new reference_filter_result_t[std::min(lenA, lenB)];
- }
- }
-
- while (true) {
- while (*A < *B) {
- SKIP_FIRST_COMPARE:
- if (++A == endA) {
- result.count = out - result.docs;
- return;
- }
- }
- while (*A > *B) {
- if (++B == endB) {
- result.count = out - result.docs;
- return;
- }
- }
- if (*A == *B) {
- *out = *A;
-
- // Copy the references of the document from every collection into result.
- for (auto const& item: a.reference_filter_results) {
- result.reference_filter_results[item.first][out - result.docs] = item.second[A - a.docs];
- }
- for (auto const& item: b.reference_filter_results) {
- result.reference_filter_results[item.first][out - result.docs] = item.second[B - b.docs];
- }
-
- out++;
-
- if (++A == endA || ++B == endB) {
- result.count = out - result.docs;
- return;
- }
- } else {
- goto SKIP_FIRST_COMPARE;
- }
- }
-}
-
-void filter_result_t::or_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {
- if (a.count == 0 && b.count == 0) {
- return;
- }
-
- // If either one of a or b does not have any matches, copy other into result.
- if (a.count == 0) {
- result = b;
- return;
- }
- if (b.count == 0) {
- result = a;
- return;
- }
-
- size_t indexA = 0, indexB = 0, res_index = 0, lenA = a.count, lenB = b.count;
- result.docs = new uint32_t[lenA + lenB];
-
- // Add an entry of references in the result for each unique collection in a and b.
- for (auto const& item: a.reference_filter_results) {
- if (result.reference_filter_results.count(item.first) == 0) {
- result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];
- }
- }
- for (auto const& item: b.reference_filter_results) {
- if (result.reference_filter_results.count(item.first) == 0) {
- result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];
- }
- }
-
- while (indexA < lenA && indexB < lenB) {
- if (a.docs[indexA] < b.docs[indexB]) {
- // check for duplicate
- if (res_index == 0 || result.docs[res_index - 1] != a.docs[indexA]) {
- result.docs[res_index] = a.docs[indexA];
- res_index++;
- }
-
- // Copy references of the last result document from every collection in a.
- for (auto const& item: a.reference_filter_results) {
- result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];
- }
-
- indexA++;
- } else {
- if (res_index == 0 || result.docs[res_index - 1] != b.docs[indexB]) {
- result.docs[res_index] = b.docs[indexB];
- res_index++;
- }
-
- for (auto const& item: b.reference_filter_results) {
- result.reference_filter_results[item.first][res_index - 1] = item.second[indexB];
- }
-
- indexB++;
- }
- }
-
- while (indexA < lenA) {
- if (res_index == 0 || result.docs[res_index - 1] != a.docs[indexA]) {
- result.docs[res_index] = a.docs[indexA];
- res_index++;
- }
-
- for (auto const& item: a.reference_filter_results) {
- result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];
- }
-
- indexA++;
- }
-
- while (indexB < lenB) {
- if(res_index == 0 || result.docs[res_index - 1] != b.docs[indexB]) {
- result.docs[res_index] = b.docs[indexB];
- res_index++;
- }
-
- for (auto const& item: b.reference_filter_results) {
- result.reference_filter_results[item.first][res_index - 1] = item.second[indexB];
- }
-
- indexB++;
- }
-
- result.count = res_index;
-
- // shrink fit
- auto out = new uint32_t[res_index];
- memcpy(out, result.docs, res_index * sizeof(uint32_t));
- delete[] result.docs;
- result.docs = out;
-
- for (auto &item: result.reference_filter_results) {
- auto out_references = new reference_filter_result_t[res_index];
-
- for (uint32_t i = 0; i < result.count; i++) {
- out_references[i] = item.second[i];
- }
- delete[] item.second;
- item.second = out_references;
- }
-}
diff --git a/src/filter.cpp b/src/filter.cpp
index 77ef5d5d..95fbfefc 100644
--- a/src/filter.cpp
+++ b/src/filter.cpp
@@ -1,641 +1,573 @@
#include <collection_manager.h>
#include <posting.h>
#include <timsort.hpp>
+#include <stack>
#include "filter.h"
-void filter_result_iterator_t::and_filter_iterators() {
- while (left_it->is_valid && right_it->is_valid) {
- while (left_it->seq_id < right_it->seq_id) {
- left_it->next();
- if (!left_it->is_valid) {
- is_valid = false;
- return;
- }
- }
-
- while (left_it->seq_id > right_it->seq_id) {
- right_it->next();
- if (!right_it->is_valid) {
- is_valid = false;
- return;
- }
- }
-
- if (left_it->seq_id == right_it->seq_id) {
- seq_id = left_it->seq_id;
- reference.clear();
+Option<bool> filter::validate_numerical_filter_value(field _field, const string &raw_value) {
+ if(_field.is_int32() && !StringUtils::is_int32_t(raw_value)) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not an int32.");
+ }
- for (const auto& item: left_it->reference) {
- reference[item.first] = item.second;
- }
- for (const auto& item: right_it->reference) {
- reference[item.first] = item.second;
- }
+ else if(_field.is_int64() && !StringUtils::is_int64_t(raw_value)) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not an int64.");
+ }
- return;
- }
+ else if(_field.is_float() && !StringUtils::is_float(raw_value)) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not a float.");
}
- is_valid = false;
+ return Option<bool>(true);
}
-void filter_result_iterator_t::or_filter_iterators() {
- if (left_it->is_valid && right_it->is_valid) {
- if (left_it->seq_id < right_it->seq_id) {
- seq_id = left_it->seq_id;
- reference.clear();
-
- for (const auto& item: left_it->reference) {
- reference[item.first] = item.second;
- }
-
- return;
- }
-
- if (left_it->seq_id > right_it->seq_id) {
- seq_id = right_it->seq_id;
- reference.clear();
-
- for (const auto& item: right_it->reference) {
- reference[item.first] = item.second;
- }
-
- return;
- }
-
- seq_id = left_it->seq_id;
- reference.clear();
-
- for (const auto& item: left_it->reference) {
- reference[item.first] = item.second;
- }
- for (const auto& item: right_it->reference) {
- reference[item.first] = item.second;
- }
+Option<NUM_COMPARATOR> filter::extract_num_comparator(string &comp_and_value) {
+ auto num_comparator = EQUALS;
- return;
+ if(StringUtils::is_integer(comp_and_value) || StringUtils::is_float(comp_and_value)) {
+ num_comparator = EQUALS;
}
- if (left_it->is_valid) {
- seq_id = left_it->seq_id;
- reference.clear();
-
- for (const auto& item: left_it->reference) {
- reference[item.first] = item.second;
- }
-
- return;
+ // the ordering is important - we have to compare 2-letter operators first
+ else if(comp_and_value.compare(0, 2, "<=") == 0) {
+ num_comparator = LESS_THAN_EQUALS;
}
- if (right_it->is_valid) {
- seq_id = right_it->seq_id;
- reference.clear();
-
- for (const auto& item: right_it->reference) {
- reference[item.first] = item.second;
- }
-
- return;
+ else if(comp_and_value.compare(0, 2, ">=") == 0) {
+ num_comparator = GREATER_THAN_EQUALS;
}
- is_valid = false;
-}
-
-void filter_result_iterator_t::doc_matching_string_filter() {
- // If none of the filter value iterators are valid, mark this node as invalid.
- bool one_is_valid = false;
-
- // Since we do OR between filter values, the lowest seq_id id from all is selected.
- uint32_t lowest_id = UINT32_MAX;
-
- for (auto& filter_value_tokens : posting_list_iterators) {
- // Perform AND between tokens of a filter value.
- bool tokens_iter_is_valid;
- posting_list_t::intersect(filter_value_tokens, tokens_iter_is_valid);
-
- one_is_valid = tokens_iter_is_valid || one_is_valid;
-
- if (tokens_iter_is_valid && filter_value_tokens[0].id() < lowest_id) {
- lowest_id = filter_value_tokens[0].id();
- }
+ else if(comp_and_value.compare(0, 2, "!=") == 0) {
+ num_comparator = NOT_EQUALS;
}
- if (one_is_valid) {
- seq_id = lowest_id;
+ else if(comp_and_value.compare(0, 1, "<") == 0) {
+ num_comparator = LESS_THAN;
}
- is_valid = one_is_valid;
-}
-
-void filter_result_iterator_t::next() {
- if (!is_valid) {
- return;
+ else if(comp_and_value.compare(0, 1, ">") == 0) {
+ num_comparator = GREATER_THAN;
}
- if (filter_node->isOperator) {
- // Advance the subtrees and then apply operators to arrive at the next valid doc.
- if (filter_node->filter_operator == AND) {
- left_it->next();
- right_it->next();
- and_filter_iterators();
- } else {
- if (left_it->seq_id == seq_id && right_it->seq_id == seq_id) {
- left_it->next();
- right_it->next();
- } else if (left_it->seq_id == seq_id) {
- left_it->next();
- } else {
- right_it->next();
- }
+ else if(comp_and_value.find("..") != std::string::npos) {
+ num_comparator = RANGE_INCLUSIVE;
+ }
- or_filter_iterators();
- }
+ else {
+ return Option<NUM_COMPARATOR>(400, "Numerical field has an invalid comparator.");
+ }
- return;
+ if(num_comparator == LESS_THAN || num_comparator == GREATER_THAN) {
+ comp_and_value = comp_and_value.substr(1);
+ } else if(num_comparator == LESS_THAN_EQUALS || num_comparator == GREATER_THAN_EQUALS || num_comparator == NOT_EQUALS) {
+ comp_and_value = comp_and_value.substr(2);
}
- const filter a_filter = filter_node->filter_exp;
+ comp_and_value = StringUtils::trim(comp_and_value);
- bool is_referenced_filter = !a_filter.referenced_collection_name.empty();
- if (is_referenced_filter) {
- if (++result_index >= filter_result.count) {
- is_valid = false;
- return;
- }
+ return Option<NUM_COMPARATOR>(num_comparator);
+}
- seq_id = filter_result.docs[result_index];
- reference.clear();
- for (auto const& item: filter_result.reference_filter_results) {
- reference[item.first] = item.second[result_index];
- }
+Option<bool> filter::parse_geopoint_filter_value(std::string& raw_value,
+ const std::string& format_err_msg,
+ std::string& processed_filter_val,
+ NUM_COMPARATOR& num_comparator) {
+
+ num_comparator = LESS_THAN_EQUALS;
- return;
+ if(!(raw_value[0] == '(' && raw_value[raw_value.size() - 1] == ')')) {
+ return Option<bool>(400, format_err_msg);
}
- if (a_filter.field_name == "id") {
- if (++result_index >= filter_result.count) {
- is_valid = false;
- return;
- }
+ std::vector<std::string> filter_values;
+ auto raw_val_without_paran = raw_value.substr(1, raw_value.size() - 2);
+ StringUtils::split(raw_val_without_paran, filter_values, ",");
- seq_id = filter_result.docs[result_index];
- return;
- }
+ // we will end up with: "10.45 34.56 2 km" or "10.45 34.56 2mi" or a geo polygon
- if (!index->field_is_indexed(a_filter.field_name)) {
- is_valid = false;
- return;
+ if(filter_values.size() < 3) {
+ return Option<bool>(400, format_err_msg);
}
- field f = index->search_schema.at(a_filter.field_name);
-
- if (f.is_string()) {
- // Advance all the filter values that are at doc. Then find the next doc.
- for (uint32_t i = 0; i < posting_list_iterators.size(); i++) {
- auto& filter_value_tokens = posting_list_iterators[i];
+ // do validation: format should match either a point + radius or polygon
- if (filter_value_tokens[0].valid() && filter_value_tokens[0].id() == seq_id) {
- for (auto& iter: filter_value_tokens) {
- iter.next();
- }
- }
+ size_t num_floats = 0;
+ for(const auto& fvalue: filter_values) {
+ if(StringUtils::is_float(fvalue)) {
+ num_floats++;
}
-
- doc_matching_string_filter();
- return;
- }
-}
-
-void filter_result_iterator_t::init() {
- if (filter_node == nullptr) {
- return;
}
- if (filter_node->isOperator) {
- if (filter_node->filter_operator == AND) {
- and_filter_iterators();
- } else {
- or_filter_iterators();
+ bool is_polygon = (num_floats == filter_values.size());
+ if(!is_polygon) {
+ // we have to ensure that this is a point + radius match
+ if(!StringUtils::is_float(filter_values[0]) || !StringUtils::is_float(filter_values[1])) {
+ return Option<bool>(400, format_err_msg);
}
- return;
+ if(filter_values[0] == "nan" || filter_values[0] == "NaN" ||
+ filter_values[1] == "nan" || filter_values[1] == "NaN") {
+ return Option<bool>(400, format_err_msg);
+ }
}
- const filter a_filter = filter_node->filter_exp;
-
- bool is_referenced_filter = !a_filter.referenced_collection_name.empty();
- if (is_referenced_filter) {
- // Apply filter on referenced collection and get the sequence ids of current collection from the filtered documents.
- auto& cm = CollectionManager::get_instance();
- auto collection = cm.get_collection(a_filter.referenced_collection_name);
- if (collection == nullptr) {
- status = Option<bool>(400, "Referenced collection `" + a_filter.referenced_collection_name + "` not found.");
- is_valid = false;
- return;
+ if(is_polygon) {
+ processed_filter_val = raw_val_without_paran;
+ } else {
+ // point + radius
+ // filter_values[2] is distance, get the unit, validate it and split on that
+ if(filter_values[2].size() < 2) {
+ return Option<bool>(400, "Unit must be either `km` or `mi`.");
}
- auto reference_filter_op = collection->get_reference_filter_ids(a_filter.field_name,
- filter_result,
- collection_name);
- if (!reference_filter_op.ok()) {
- status = Option<bool>(400, "Failed to apply reference filter on `" + a_filter.referenced_collection_name
- + "` collection: " + reference_filter_op.error());
- is_valid = false;
- return;
+ std::string unit = filter_values[2].substr(filter_values[2].size()-2, 2);
+
+ if(unit != "km" && unit != "mi") {
+ return Option<bool>(400, "Unit must be either `km` or `mi`.");
}
- is_valid = filter_result.count > 0;
- return;
- }
+ std::vector<std::string> dist_values;
+ StringUtils::split(filter_values[2], dist_values, unit);
- if (a_filter.field_name == "id") {
- if (a_filter.values.empty()) {
- is_valid = false;
- return;
+ if(dist_values.size() != 1) {
+ return Option<bool>(400, format_err_msg);
}
- // we handle `ids` separately
- std::vector<uint32_t> result_ids;
- for (const auto& id_str : a_filter.values) {
- result_ids.push_back(std::stoul(id_str));
+ if(!StringUtils::is_float(dist_values[0])) {
+ return Option<bool>(400, format_err_msg);
}
- std::sort(result_ids.begin(), result_ids.end());
-
- filter_result.count = result_ids.size();
- filter_result.docs = new uint32_t[result_ids.size()];
- std::copy(result_ids.begin(), result_ids.end(), filter_result.docs);
+ processed_filter_val = filter_values[0] + ", " + filter_values[1] + ", " + // co-ords
+ dist_values[0] + ", " + unit; // X km
}
- if (!index->field_is_indexed(a_filter.field_name)) {
- is_valid = false;
- return;
- }
-
- field f = index->search_schema.at(a_filter.field_name);
-
- if (f.is_string()) {
- art_tree* t = index->search_index.at(a_filter.field_name);
-
- for (const std::string& filter_value : a_filter.values) {
- std::vector<void*> posting_lists;
-
- // there could be multiple tokens in a filter value, which we have to treat as ANDs
- // e.g. country: South Africa
- Tokenizer tokenizer(filter_value, true, false, f.locale, index->symbols_to_index, index->token_separators);
+ return Option<bool>(true);
+}
- std::string str_token;
- size_t token_index = 0;
- std::vector<std::string> str_tokens;
+bool isOperator(const std::string& expression) {
+ return expression == "&&" || expression == "||";
+}
- while (tokenizer.next(str_token, token_index)) {
- str_tokens.push_back(str_token);
+// https://en.wikipedia.org/wiki/Shunting_yard_algorithm
+Option<bool> toPostfix(std::queue<std::string>& tokens, std::queue<std::string>& postfix) {
+ std::stack<std::string> operatorStack;
- art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
- str_token.length()+1);
- if (leaf == nullptr) {
- continue;
- }
+ while (!tokens.empty()) {
+ auto expression = tokens.front();
+ tokens.pop();
- posting_lists.push_back(leaf->values);
+ if (isOperator(expression)) {
+ // We only have two operators &&, || having the same precedence and both being left associative.
+ while (!operatorStack.empty() && operatorStack.top() != "(") {
+ postfix.push(operatorStack.top());
+ operatorStack.pop();
}
- if (posting_lists.size() != str_tokens.size()) {
- continue;
+ operatorStack.push(expression);
+ } else if (expression == "(") {
+ operatorStack.push(expression);
+ } else if (expression == ")") {
+ while (!operatorStack.empty() && operatorStack.top() != "(") {
+ postfix.push(operatorStack.top());
+ operatorStack.pop();
}
- std::vector<posting_list_t*> plists;
- posting_t::to_expanded_plists(posting_lists, plists, expanded_plists);
-
- posting_list_iterators.emplace_back(std::vector<posting_list_t::iterator_t>());
-
- for (auto const& plist: plists) {
- posting_list_iterators.back().push_back(plist->new_iterator());
+ if (operatorStack.empty() || operatorStack.top() != "(") {
+ return Option<bool>(400, "Could not parse the filter query: unbalanced parentheses.");
}
- }
-
- doc_matching_string_filter();
- return;
- }
-}
-
-bool filter_result_iterator_t::valid() {
- if (!is_valid) {
- return false;
- }
-
- if (filter_node->isOperator) {
- if (filter_node->filter_operator == AND) {
- is_valid = left_it->valid() && right_it->valid();
- return is_valid;
+ operatorStack.pop();
} else {
- is_valid = left_it->valid() || right_it->valid();
- return is_valid;
+ postfix.push(expression);
}
}
- const filter a_filter = filter_node->filter_exp;
-
- if (!a_filter.referenced_collection_name.empty() || a_filter.field_name == "id") {
- is_valid = result_index < filter_result.count;
- return is_valid;
- }
-
- if (!index->field_is_indexed(a_filter.field_name)) {
- is_valid = false;
- return is_valid;
+ while (!operatorStack.empty()) {
+ if (operatorStack.top() == "(") {
+ return Option<bool>(400, "Could not parse the filter query: unbalanced parentheses.");
+ }
+ postfix.push(operatorStack.top());
+ operatorStack.pop();
}
- field f = index->search_schema.at(a_filter.field_name);
-
- if (f.is_string()) {
- bool one_is_valid = false;
- for (auto& filter_value_tokens: posting_list_iterators) {
- posting_list_t::intersect(filter_value_tokens, one_is_valid);
+ return Option<bool>(true);
+}
- if (one_is_valid) {
- break;
+Option<bool> toMultiValueNumericFilter(std::string& raw_value, filter& filter_exp, const field& _field) {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+ filter_exp = {_field.name, {}, {}};
+ for (std::string& filter_value: filter_values) {
+ Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(filter_value);
+ if (!op_comparator.ok()) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());
+ }
+ if (op_comparator.get() == RANGE_INCLUSIVE) {
+ // split the value around range operator to extract bounds
+ std::vector<std::string> range_values;
+ StringUtils::split(filter_value, range_values, filter::RANGE_OPERATOR());
+ for (const std::string& range_value: range_values) {
+ auto validate_op = filter::validate_numerical_filter_value(_field, range_value);
+ if (!validate_op.ok()) {
+ return validate_op;
+ }
+ filter_exp.values.push_back(range_value);
+ filter_exp.comparators.push_back(op_comparator.get());
+ }
+ } else {
+ auto validate_op = filter::validate_numerical_filter_value(_field, filter_value);
+ if (!validate_op.ok()) {
+ return validate_op;
}
+ filter_exp.values.push_back(filter_value);
+ filter_exp.comparators.push_back(op_comparator.get());
}
-
- is_valid = one_is_valid;
- return is_valid;
}
- return false;
+ return Option<bool>(true);
}
-void filter_result_iterator_t::skip_to(uint32_t id) {
- if (!is_valid) {
- return;
- }
-
- if (filter_node->isOperator) {
- // Skip the subtrees to id and then apply operators to arrive at the next valid doc.
- left_it->skip_to(id);
- right_it->skip_to(id);
-
- if (filter_node->filter_operator == AND) {
- and_filter_iterators();
+Option<bool> toFilter(const std::string expression,
+ filter& filter_exp,
+ const tsl::htrie_map<char, field>& search_schema,
+ const Store* store,
+ const std::string& doc_id_prefix) {
+ // split into [field_name, value]
+ size_t found_index = expression.find(':');
+ if (found_index == std::string::npos) {
+ return Option<bool>(400, "Could not parse the filter query.");
+ }
+ std::string&& field_name = expression.substr(0, found_index);
+ StringUtils::trim(field_name);
+ if (field_name == "id") {
+ std::string&& raw_value = expression.substr(found_index + 1, std::string::npos);
+ StringUtils::trim(raw_value);
+ std::string empty_filter_err = "Error with filter field `id`: Filter value cannot be empty.";
+ if (raw_value.empty()) {
+ return Option<bool>(400, empty_filter_err);
+ }
+ filter_exp = {field_name, {}, {}};
+ NUM_COMPARATOR id_comparator = EQUALS;
+ size_t filter_value_index = 0;
+ if (raw_value[0] == '=') {
+ id_comparator = EQUALS;
+ while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ } else if (raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
+ return Option<bool>(400, "Not equals filtering is not supported on the `id` field.");
+ }
+ if (filter_value_index != 0) {
+ raw_value = raw_value.substr(filter_value_index);
+ }
+ if (raw_value.empty()) {
+ return Option<bool>(400, empty_filter_err);
+ }
+ if (raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> doc_ids;
+ StringUtils::split_to_values(raw_value.substr(1, raw_value.size() - 2), doc_ids);
+ for (std::string& doc_id: doc_ids) {
+ // we have to convert the doc_id to seq id
+ std::string seq_id_str;
+ StoreStatus seq_id_status = store->get(doc_id_prefix + doc_id, seq_id_str);
+ if (seq_id_status != StoreStatus::FOUND) {
+ continue;
+ }
+ filter_exp.values.push_back(seq_id_str);
+ filter_exp.comparators.push_back(id_comparator);
+ }
} else {
- or_filter_iterators();
+ std::vector<std::string> doc_ids;
+ StringUtils::split_to_values(raw_value, doc_ids); // to handle backticks
+ std::string seq_id_str;
+ StoreStatus seq_id_status = store->get(doc_id_prefix + doc_ids[0], seq_id_str);
+ if (seq_id_status == StoreStatus::FOUND) {
+ filter_exp.values.push_back(seq_id_str);
+ filter_exp.comparators.push_back(id_comparator);
+ }
}
-
- return;
+ return Option<bool>(true);
}
- const filter a_filter = filter_node->filter_exp;
-
- bool is_referenced_filter = !a_filter.referenced_collection_name.empty();
- if (is_referenced_filter) {
- while (filter_result.docs[result_index] < id && ++result_index < filter_result.count);
-
- if (result_index >= filter_result.count) {
- is_valid = false;
- return;
- }
+ auto field_it = search_schema.find(field_name);
- seq_id = filter_result.docs[result_index];
- reference.clear();
- for (auto const& item: filter_result.reference_filter_results) {
- reference[item.first] = item.second[result_index];
- }
-
- return;
+ if (field_it == search_schema.end()) {
+ return Option<bool>(404, "Could not find a filter field named `" + field_name + "` in the schema.");
}
- if (a_filter.field_name == "id") {
- while (filter_result.docs[result_index] < id && ++result_index < filter_result.count);
-
- if (result_index >= filter_result.count) {
- is_valid = false;
- return;
- }
-
- seq_id = filter_result.docs[result_index];
- return;
+ if (field_it->num_dim > 0) {
+ return Option<bool>(404, "Cannot filter on vector field `" + field_name + "`.");
}
- if (!index->field_is_indexed(a_filter.field_name)) {
- is_valid = false;
- return;
+ const field& _field = field_it.value();
+ std::string&& raw_value = expression.substr(found_index + 1, std::string::npos);
+ StringUtils::trim(raw_value);
+ // skip past optional `:=` operator, which has no meaning for non-string fields
+ if (!_field.is_string() && raw_value[0] == '=') {
+ size_t filter_value_index = 0;
+ while (raw_value[++filter_value_index] == ' ');
+ raw_value = raw_value.substr(filter_value_index);
}
-
- field f = index->search_schema.at(a_filter.field_name);
-
- if (f.is_string()) {
- // Skip all the token iterators and find a new match.
- for (auto& filter_value_tokens : posting_list_iterators) {
- for (auto& token: filter_value_tokens) {
- // We perform AND on tokens. Short-circuiting here.
- if (!token.valid()) {
- break;
+ if (_field.is_integer() || _field.is_float()) {
+ // could be a single value or a list
+ if (raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ Option<bool> op = toMultiValueNumericFilter(raw_value, filter_exp, _field);
+ if (!op.ok()) {
+ return op;
+ }
+ } else {
+ Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);
+ if (!op_comparator.ok()) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());
+ }
+ if (op_comparator.get() == RANGE_INCLUSIVE) {
+ // split the value around range operator to extract bounds
+ std::vector<std::string> range_values;
+ StringUtils::split(raw_value, range_values, filter::RANGE_OPERATOR());
+ filter_exp.field_name = field_name;
+ for (const std::string& range_value: range_values) {
+ auto validate_op = filter::validate_numerical_filter_value(_field, range_value);
+ if (!validate_op.ok()) {
+ return validate_op;
+ }
+ filter_exp.values.push_back(range_value);
+ filter_exp.comparators.push_back(op_comparator.get());
}
-
- token.skip_to(id);
+ } else if (op_comparator.get() == NOT_EQUALS && raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ Option<bool> op = toMultiValueNumericFilter(raw_value, filter_exp, _field);
+ if (!op.ok()) {
+ return op;
+ }
+ filter_exp.apply_not_equals = true;
+ } else {
+ auto validate_op = filter::validate_numerical_filter_value(_field, raw_value);
+ if (!validate_op.ok()) {
+ return validate_op;
+ }
+ filter_exp = {field_name, {raw_value}, {op_comparator.get()}};
}
}
-
- doc_matching_string_filter();
- return;
- }
-}
-
-int filter_result_iterator_t::valid(uint32_t id) {
- if (!is_valid) {
- return -1;
- }
-
- if (filter_node->isOperator) {
- auto left_valid = left_it->valid(id), right_valid = right_it->valid(id);
-
- if (filter_node->filter_operator == AND) {
- is_valid = left_it->is_valid && right_it->is_valid;
-
- if (left_valid < 1 || right_valid < 1) {
- if (left_valid == -1 || right_valid == -1) {
- return -1;
+ } else if (_field.is_bool()) {
+ NUM_COMPARATOR bool_comparator = EQUALS;
+ size_t filter_value_index = 0;
+ if (raw_value[0] == '=') {
+ bool_comparator = EQUALS;
+ while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ } else if (raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
+ bool_comparator = NOT_EQUALS;
+ filter_value_index++;
+ while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ }
+ if (filter_value_index != 0) {
+ raw_value = raw_value.substr(filter_value_index);
+ }
+ if (filter_value_index == raw_value.size()) {
+ return Option<bool>(400, "Error with filter field `" + _field.name +
+ "`: Filter value cannot be empty.");
+ }
+ if (raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+ filter_exp = {field_name, {}, {}};
+ for (std::string& filter_value: filter_values) {
+ if (filter_value != "true" && filter_value != "false") {
+ return Option<bool>(400, "Values of filter field `" + _field.name +
+ "`: must be `true` or `false`.");
}
-
- return 0;
+ filter_value = (filter_value == "true") ? "1" : "0";
+ filter_exp.values.push_back(filter_value);
+ filter_exp.comparators.push_back(bool_comparator);
}
-
- return 1;
} else {
- is_valid = left_it->is_valid || right_it->is_valid;
-
- if (left_valid < 1 && right_valid < 1) {
- if (left_valid == -1 && right_valid == -1) {
- return -1;
+ if (raw_value != "true" && raw_value != "false") {
+ return Option<bool>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");
+ }
+ std::string bool_value = (raw_value == "true") ? "1" : "0";
+ filter_exp = {field_name, {bool_value}, {bool_comparator}};
+ }
+ } else if (_field.is_geopoint()) {
+ filter_exp = {field_name, {}, {}};
+ const std::string& format_err_msg = "Value of filter field `" + _field.name +
+ "`: must be in the `(-44.50, 170.29, 0.75 km)` or "
+ "(56.33, -65.97, 23.82, -127.82) format.";
+ NUM_COMPARATOR num_comparator;
+ // could be a single value or a list
+ if (raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, "),");
+ for (std::string& filter_value: filter_values) {
+ filter_value += ")";
+ std::string processed_filter_val;
+ auto parse_op = filter::parse_geopoint_filter_value(filter_value, format_err_msg, processed_filter_val,
+ num_comparator);
+ if (!parse_op.ok()) {
+ return parse_op;
}
-
- return 0;
+ filter_exp.values.push_back(processed_filter_val);
+ filter_exp.comparators.push_back(num_comparator);
}
-
- return 1;
- }
- }
-
- if (filter_node->filter_exp.apply_not_equals) {
- // Even when iterator becomes invalid, we keep it marked as valid since we are evaluating not equals.
- if (!valid()) {
- is_valid = true;
- return 1;
- }
-
- skip_to(id);
-
- if (!is_valid) {
- is_valid = true;
- return 1;
+ } else {
+ // single value, e.g. (10.45, 34.56, 2 km)
+ std::string processed_filter_val;
+ auto parse_op = filter::parse_geopoint_filter_value(raw_value, format_err_msg, processed_filter_val,
+ num_comparator);
+ if (!parse_op.ok()) {
+ return parse_op;
+ }
+ filter_exp.values.push_back(processed_filter_val);
+ filter_exp.comparators.push_back(num_comparator);
+ }
+ } else if (_field.is_string()) {
+ size_t filter_value_index = 0;
+ NUM_COMPARATOR str_comparator = CONTAINS;
+ if (raw_value[0] == '=') {
+ // string filter should be evaluated in strict "equals" mode
+ str_comparator = EQUALS;
+ while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ } else if (raw_value.size() >= 2 && raw_value[0] == '!' && raw_value[1] == '=') {
+ str_comparator = NOT_EQUALS;
+ filter_value_index++;
+ while (++filter_value_index < raw_value.size() && raw_value[filter_value_index] == ' ');
+ }
+ if (filter_value_index == raw_value.size()) {
+ return Option<bool>(400, "Error with filter field `" + _field.name +
+ "`: Filter value cannot be empty.");
+ }
+ if (raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split_to_values(
+ raw_value.substr(filter_value_index + 1, raw_value.size() - filter_value_index - 2), filter_values);
+ filter_exp = {field_name, filter_values, {str_comparator}};
+ } else {
+ filter_exp = {field_name, {raw_value.substr(filter_value_index)}, {str_comparator}};
}
- return seq_id != id ? 1 : 0;
- }
-
- skip_to(id);
- return is_valid ? (seq_id == id ? 1 : 0) : -1;
-}
-
-Option<bool> filter_result_iterator_t::init_status() {
- if (filter_node != nullptr && filter_node->isOperator) {
- auto left_status = left_it->init_status();
-
- return !left_status.ok() ? left_status : right_it->init_status();
+ filter_exp.apply_not_equals = (str_comparator == NOT_EQUALS);
+ } else {
+ return Option<bool>(400, "Error with filter field `" + _field.name +
+ "`: Unidentified field data type, see docs for supported data types.");
}
- return status;
+ return Option<bool>(true);
}
-bool filter_result_iterator_t::contains_atleast_one(const void *obj) {
- if(IS_COMPACT_POSTING(obj)) {
- compact_posting_list_t* list = COMPACT_POSTING_PTR(obj);
-
- size_t i = 0;
- while(i < list->length && valid()) {
- size_t num_existing_offsets = list->id_offsets[i];
- size_t existing_id = list->id_offsets[i + num_existing_offsets + 1];
-
- if (existing_id == seq_id) {
- return true;
+// https://stackoverflow.com/a/423914/11218270
+Option<bool> toParseTree(std::queue<std::string>& postfix, filter_node_t*& root,
+ const tsl::htrie_map<char, field>& search_schema,
+ const Store* store,
+ const std::string& doc_id_prefix) {
+ std::stack<filter_node_t*> nodeStack;
+ bool is_successful = true;
+ std::string error_message;
+
+ filter_node_t *filter_node = nullptr;
+
+ while (!postfix.empty()) {
+ const std::string expression = postfix.front();
+ postfix.pop();
+
+ if (isOperator(expression)) {
+ if (nodeStack.empty()) {
+ is_successful = false;
+ error_message = "Could not parse the filter query: unbalanced `" + expression + "` operands.";
+ break;
}
+ auto operandB = nodeStack.top();
+ nodeStack.pop();
- // advance smallest value
- if (existing_id < seq_id) {
- i += num_existing_offsets + 2;
- } else {
- skip_to(existing_id);
+ if (nodeStack.empty()) {
+ delete operandB;
+ is_successful = false;
+ error_message = "Could not parse the filter query: unbalanced `" + expression + "` operands.";
+ break;
}
- }
- } else {
- auto list = (posting_list_t*)(obj);
- posting_list_t::iterator_t it = list->new_iterator();
-
- while(it.valid() && valid()) {
- uint32_t id = it.id();
+ auto operandA = nodeStack.top();
+ nodeStack.pop();
- if(id == seq_id) {
- return true;
- }
+ filter_node = new filter_node_t(expression == "&&" ? AND : OR, operandA, operandB);
+ } else {
+ filter filter_exp;
+
+ // Expected value: $Collection(...)
+ bool is_referenced_filter = (expression[0] == '$' && expression[expression.size() - 1] == ')');
+ if (is_referenced_filter) {
+ size_t parenthesis_index = expression.find('(');
+
+ std::string collection_name = expression.substr(1, parenthesis_index - 1);
+ auto &cm = CollectionManager::get_instance();
+ auto collection = cm.get_collection(collection_name);
+ if (collection == nullptr) {
+ is_successful = false;
+ error_message = "Referenced collection `" + collection_name + "` not found.";
+ break;
+ }
- if(id < seq_id) {
- it.skip_to(seq_id);
+ filter_exp = {expression.substr(parenthesis_index + 1, expression.size() - parenthesis_index - 2)};
+ filter_exp.referenced_collection_name = collection_name;
} else {
- skip_to(id);
+ Option<bool> toFilter_op = toFilter(expression, filter_exp, search_schema, store, doc_id_prefix);
+ if (!toFilter_op.ok()) {
+ is_successful = false;
+ error_message = toFilter_op.error();
+ break;
+ }
}
- }
- }
-
- return false;
-}
-
-void filter_result_iterator_t::reset() {
- if (filter_node == nullptr) {
- return;
- }
-
- if (filter_node->isOperator) {
- // Reset the subtrees then apply operators to arrive at the first valid doc.
- left_it->reset();
- right_it->reset();
- if (filter_node->filter_operator == AND) {
- and_filter_iterators();
- } else {
- or_filter_iterators();
+ filter_node = new filter_node_t(filter_exp);
}
- return;
- }
-
- const filter a_filter = filter_node->filter_exp;
-
- bool is_referenced_filter = !a_filter.referenced_collection_name.empty();
- if (is_referenced_filter || a_filter.field_name == "id") {
- result_index = 0;
- is_valid = filter_result.count > 0;
- return;
+ nodeStack.push(filter_node);
}
- if (!index->field_is_indexed(a_filter.field_name)) {
- return;
- }
-
- field f = index->search_schema.at(a_filter.field_name);
-
- if (f.is_string()) {
- posting_list_iterators.clear();
- for(auto expanded_plist: expanded_plists) {
- delete expanded_plist;
+ if (!is_successful) {
+ while (!nodeStack.empty()) {
+ auto filterNode = nodeStack.top();
+ delete filterNode;
+ nodeStack.pop();
}
- expanded_plists.clear();
- init();
- return;
+ return Option<bool>(400, error_message);
}
-}
-uint32_t filter_result_iterator_t::to_filter_id_array(uint32_t*& filter_array) {
- if (!valid()) {
- return 0;
+ if (nodeStack.empty()) {
+ return Option<bool>(400, "Filter query cannot be empty.");
}
+ root = nodeStack.top();
- std::vector<uint32_t> filter_ids;
- do {
- filter_ids.push_back(seq_id);
- next();
- } while (valid());
-
- filter_array = new uint32_t[filter_ids.size()];
- std::copy(filter_ids.begin(), filter_ids.end(), filter_array);
-
- return filter_ids.size();
+ return Option<bool>(true);
}
-uint32_t filter_result_iterator_t::and_scalar(const uint32_t* A, const uint32_t& lenA, uint32_t*& results) {
- if (!valid()) {
- return 0;
+Option<bool> filter::parse_filter_query(const std::string& filter_query,
+ const tsl::htrie_map<char, field>& search_schema,
+ const Store* store,
+ const std::string& doc_id_prefix,
+ filter_node_t*& root) {
+ auto _filter_query = filter_query;
+ StringUtils::trim(_filter_query);
+ if (_filter_query.empty()) {
+ return Option<bool>(true);
}
- std::vector<uint32_t> filter_ids;
- for (uint32_t i = 0; i < lenA; i++) {
- auto result = valid(A[i]);
-
- if (result == -1) {
- break;
- }
+ std::queue<std::string> tokens;
+ Option<bool> tokenize_op = StringUtils::tokenize_filter_query(filter_query, tokens);
+ if (!tokenize_op.ok()) {
+ return tokenize_op;
+ }
- if (result == 1) {
- filter_ids.push_back(A[i]);
- }
+ if (tokens.size() > 100) {
+ return Option<bool>(400, "Filter expression is not valid.");
}
- if (filter_ids.empty()) {
- return 0;
+ std::queue<std::string> postfix;
+ Option<bool> toPostfix_op = toPostfix(tokens, postfix);
+ if (!toPostfix_op.ok()) {
+ return toPostfix_op;
}
- results = new uint32_t[filter_ids.size()];
- std::copy(filter_ids.begin(), filter_ids.end(), results);
+ Option<bool> toParseTree_op = toParseTree(postfix,
+ root,
+ search_schema,
+ store,
+ doc_id_prefix);
+ if (!toParseTree_op.ok()) {
+ return toParseTree_op;
+ }
- return filter_ids.size();
+ return Option<bool>(true);
}
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cpp
new file mode 100644
index 00000000..21a4128f
--- /dev/null
+++ b/src/filter_result_iterator.cpp
@@ -0,0 +1,901 @@
+#include "filter_result_iterator.h"
+#include "index.h"
+#include "posting.h"
+#include "collection_manager.h"
+
+void filter_result_t::and_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {
+ auto lenA = a.count, lenB = b.count;
+ if (lenA == 0 || lenB == 0) {
+ return;
+ }
+
+ result.docs = new uint32_t[std::min(lenA, lenB)];
+
+ auto A = a.docs, B = b.docs, out = result.docs;
+ const uint32_t *endA = A + lenA;
+ const uint32_t *endB = B + lenB;
+
+ // Add an entry of references in the result for each unique collection in a and b.
+ for (auto const& item: a.reference_filter_results) {
+ if (result.reference_filter_results.count(item.first) == 0) {
+ result.reference_filter_results[item.first] = new reference_filter_result_t[std::min(lenA, lenB)];
+ }
+ }
+ for (auto const& item: b.reference_filter_results) {
+ if (result.reference_filter_results.count(item.first) == 0) {
+ result.reference_filter_results[item.first] = new reference_filter_result_t[std::min(lenA, lenB)];
+ }
+ }
+
+ while (true) {
+ while (*A < *B) {
+ SKIP_FIRST_COMPARE:
+ if (++A == endA) {
+ result.count = out - result.docs;
+ return;
+ }
+ }
+ while (*A > *B) {
+ if (++B == endB) {
+ result.count = out - result.docs;
+ return;
+ }
+ }
+ if (*A == *B) {
+ *out = *A;
+
+ // Copy the references of the document from every collection into result.
+ for (auto const& item: a.reference_filter_results) {
+ result.reference_filter_results[item.first][out - result.docs] = item.second[A - a.docs];
+ }
+ for (auto const& item: b.reference_filter_results) {
+ result.reference_filter_results[item.first][out - result.docs] = item.second[B - b.docs];
+ }
+
+ out++;
+
+ if (++A == endA || ++B == endB) {
+ result.count = out - result.docs;
+ return;
+ }
+ } else {
+ goto SKIP_FIRST_COMPARE;
+ }
+ }
+}
+
+void filter_result_t::or_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {
+ if (a.count == 0 && b.count == 0) {
+ return;
+ }
+
+ // If either one of a or b does not have any matches, copy other into result.
+ if (a.count == 0) {
+ result = b;
+ return;
+ }
+ if (b.count == 0) {
+ result = a;
+ return;
+ }
+
+ size_t indexA = 0, indexB = 0, res_index = 0, lenA = a.count, lenB = b.count;
+ result.docs = new uint32_t[lenA + lenB];
+
+ // Add an entry of references in the result for each unique collection in a and b.
+ for (auto const& item: a.reference_filter_results) {
+ if (result.reference_filter_results.count(item.first) == 0) {
+ result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];
+ }
+ }
+ for (auto const& item: b.reference_filter_results) {
+ if (result.reference_filter_results.count(item.first) == 0) {
+ result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];
+ }
+ }
+
+ while (indexA < lenA && indexB < lenB) {
+ if (a.docs[indexA] < b.docs[indexB]) {
+ // check for duplicate
+ if (res_index == 0 || result.docs[res_index - 1] != a.docs[indexA]) {
+ result.docs[res_index] = a.docs[indexA];
+ res_index++;
+ }
+
+ // Copy references of the last result document from every collection in a.
+ for (auto const& item: a.reference_filter_results) {
+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];
+ }
+
+ indexA++;
+ } else {
+ if (res_index == 0 || result.docs[res_index - 1] != b.docs[indexB]) {
+ result.docs[res_index] = b.docs[indexB];
+ res_index++;
+ }
+
+ for (auto const& item: b.reference_filter_results) {
+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexB];
+ }
+
+ indexB++;
+ }
+ }
+
+ while (indexA < lenA) {
+ if (res_index == 0 || result.docs[res_index - 1] != a.docs[indexA]) {
+ result.docs[res_index] = a.docs[indexA];
+ res_index++;
+ }
+
+ for (auto const& item: a.reference_filter_results) {
+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];
+ }
+
+ indexA++;
+ }
+
+ while (indexB < lenB) {
+ if(res_index == 0 || result.docs[res_index - 1] != b.docs[indexB]) {
+ result.docs[res_index] = b.docs[indexB];
+ res_index++;
+ }
+
+ for (auto const& item: b.reference_filter_results) {
+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexB];
+ }
+
+ indexB++;
+ }
+
+ result.count = res_index;
+
+ // shrink fit
+ auto out = new uint32_t[res_index];
+ memcpy(out, result.docs, res_index * sizeof(uint32_t));
+ delete[] result.docs;
+ result.docs = out;
+
+ for (auto &item: result.reference_filter_results) {
+ auto out_references = new reference_filter_result_t[res_index];
+
+ for (uint32_t i = 0; i < result.count; i++) {
+ out_references[i] = item.second[i];
+ }
+ delete[] item.second;
+ item.second = out_references;
+ }
+}
+
+void filter_result_iterator_t::and_filter_iterators() {
+ while (left_it->is_valid && right_it->is_valid) {
+ while (left_it->seq_id < right_it->seq_id) {
+ left_it->next();
+ if (!left_it->is_valid) {
+ is_valid = false;
+ return;
+ }
+ }
+
+ while (left_it->seq_id > right_it->seq_id) {
+ right_it->next();
+ if (!right_it->is_valid) {
+ is_valid = false;
+ return;
+ }
+ }
+
+ if (left_it->seq_id == right_it->seq_id) {
+ seq_id = left_it->seq_id;
+ reference.clear();
+
+ for (const auto& item: left_it->reference) {
+ reference[item.first] = item.second;
+ }
+ for (const auto& item: right_it->reference) {
+ reference[item.first] = item.second;
+ }
+
+ return;
+ }
+ }
+
+ is_valid = false;
+}
+
+void filter_result_iterator_t::or_filter_iterators() {
+ if (left_it->is_valid && right_it->is_valid) {
+ if (left_it->seq_id < right_it->seq_id) {
+ seq_id = left_it->seq_id;
+ reference.clear();
+
+ for (const auto& item: left_it->reference) {
+ reference[item.first] = item.second;
+ }
+
+ return;
+ }
+
+ if (left_it->seq_id > right_it->seq_id) {
+ seq_id = right_it->seq_id;
+ reference.clear();
+
+ for (const auto& item: right_it->reference) {
+ reference[item.first] = item.second;
+ }
+
+ return;
+ }
+
+ seq_id = left_it->seq_id;
+ reference.clear();
+
+ for (const auto& item: left_it->reference) {
+ reference[item.first] = item.second;
+ }
+ for (const auto& item: right_it->reference) {
+ reference[item.first] = item.second;
+ }
+
+ return;
+ }
+
+ if (left_it->is_valid) {
+ seq_id = left_it->seq_id;
+ reference.clear();
+
+ for (const auto& item: left_it->reference) {
+ reference[item.first] = item.second;
+ }
+
+ return;
+ }
+
+ if (right_it->is_valid) {
+ seq_id = right_it->seq_id;
+ reference.clear();
+
+ for (const auto& item: right_it->reference) {
+ reference[item.first] = item.second;
+ }
+
+ return;
+ }
+
+ is_valid = false;
+}
+
+void filter_result_iterator_t::doc_matching_string_filter(bool field_is_array) {
+ // If none of the filter value iterators are valid, mark this node as invalid.
+ bool one_is_valid = false;
+
+ // Since we do OR between filter values, the lowest seq_id id from all is selected.
+ uint32_t lowest_id = UINT32_MAX;
+
+ if (filter_node->filter_exp.comparators[0] == EQUALS || filter_node->filter_exp.comparators[0] == NOT_EQUALS) {
+ for (auto& filter_value_tokens : posting_list_iterators) {
+ bool tokens_iter_is_valid, exact_match = false;
+ while(true) {
+ // Perform AND between tokens of a filter value.
+ posting_list_t::intersect(filter_value_tokens, tokens_iter_is_valid);
+
+ if (!tokens_iter_is_valid) {
+ break;
+ }
+
+ if (posting_list_t::has_exact_match(filter_value_tokens, field_is_array)) {
+ exact_match = true;
+ break;
+ } else {
+ // Keep advancing token iterators till exact match is not found.
+ for (auto &item: filter_value_tokens) {
+ item.next();
+ }
+ }
+ }
+
+ one_is_valid = tokens_iter_is_valid || one_is_valid;
+
+ if (tokens_iter_is_valid && exact_match && filter_value_tokens[0].id() < lowest_id) {
+ lowest_id = filter_value_tokens[0].id();
+ }
+ }
+ } else {
+ for (auto& filter_value_tokens : posting_list_iterators) {
+ // Perform AND between tokens of a filter value.
+ bool tokens_iter_is_valid;
+ posting_list_t::intersect(filter_value_tokens, tokens_iter_is_valid);
+
+ one_is_valid = tokens_iter_is_valid || one_is_valid;
+
+ if (tokens_iter_is_valid && filter_value_tokens[0].id() < lowest_id) {
+ lowest_id = filter_value_tokens[0].id();
+ }
+ }
+ }
+
+ if (one_is_valid) {
+ seq_id = lowest_id;
+ }
+
+ is_valid = one_is_valid;
+}
+
+void filter_result_iterator_t::next() {
+ if (!is_valid) {
+ return;
+ }
+
+ if (filter_node->isOperator) {
+ // Advance the subtrees and then apply operators to arrive at the next valid doc.
+ if (filter_node->filter_operator == AND) {
+ left_it->next();
+ right_it->next();
+ and_filter_iterators();
+ } else {
+ if (left_it->seq_id == seq_id && right_it->seq_id == seq_id) {
+ left_it->next();
+ right_it->next();
+ } else if (left_it->seq_id == seq_id) {
+ left_it->next();
+ } else {
+ right_it->next();
+ }
+
+ or_filter_iterators();
+ }
+
+ return;
+ }
+
+ const filter a_filter = filter_node->filter_exp;
+
+ bool is_referenced_filter = !a_filter.referenced_collection_name.empty();
+ if (is_referenced_filter) {
+ if (++result_index >= filter_result.count) {
+ is_valid = false;
+ return;
+ }
+
+ seq_id = filter_result.docs[result_index];
+ reference.clear();
+ for (auto const& item: filter_result.reference_filter_results) {
+ reference[item.first] = item.second[result_index];
+ }
+
+ return;
+ }
+
+ if (a_filter.field_name == "id") {
+ if (++result_index >= filter_result.count) {
+ is_valid = false;
+ return;
+ }
+
+ seq_id = filter_result.docs[result_index];
+ return;
+ }
+
+ if (!index->field_is_indexed(a_filter.field_name)) {
+ is_valid = false;
+ return;
+ }
+
+ field f = index->search_schema.at(a_filter.field_name);
+
+ if (f.is_string()) {
+ // Advance all the filter values that are at doc. Then find the next doc.
+ for (uint32_t i = 0; i < posting_list_iterators.size(); i++) {
+ auto& filter_value_tokens = posting_list_iterators[i];
+
+ if (filter_value_tokens[0].valid() && filter_value_tokens[0].id() == seq_id) {
+ for (auto& iter: filter_value_tokens) {
+ iter.next();
+ }
+ }
+ }
+
+ doc_matching_string_filter(f.is_array());
+ return;
+ }
+}
+
+void filter_result_iterator_t::init() {
+ if (filter_node == nullptr) {
+ return;
+ }
+
+ if (filter_node->isOperator) {
+ if (filter_node->filter_operator == AND) {
+ and_filter_iterators();
+ } else {
+ or_filter_iterators();
+ }
+
+ return;
+ }
+
+ const filter a_filter = filter_node->filter_exp;
+
+ bool is_referenced_filter = !a_filter.referenced_collection_name.empty();
+ if (is_referenced_filter) {
+ // Apply filter on referenced collection and get the sequence ids of current collection from the filtered documents.
+ auto& cm = CollectionManager::get_instance();
+ auto collection = cm.get_collection(a_filter.referenced_collection_name);
+ if (collection == nullptr) {
+ status = Option<bool>(400, "Referenced collection `" + a_filter.referenced_collection_name + "` not found.");
+ is_valid = false;
+ return;
+ }
+
+ auto reference_filter_op = collection->get_reference_filter_ids(a_filter.field_name,
+ filter_result,
+ collection_name);
+ if (!reference_filter_op.ok()) {
+ status = Option<bool>(400, "Failed to apply reference filter on `" + a_filter.referenced_collection_name
+ + "` collection: " + reference_filter_op.error());
+ is_valid = false;
+ return;
+ }
+
+ is_valid = filter_result.count > 0;
+ return;
+ }
+
+ if (a_filter.field_name == "id") {
+ if (a_filter.values.empty()) {
+ is_valid = false;
+ return;
+ }
+
+ // we handle `ids` separately
+ std::vector<uint32_t> result_ids;
+ for (const auto& id_str : a_filter.values) {
+ result_ids.push_back(std::stoul(id_str));
+ }
+
+ std::sort(result_ids.begin(), result_ids.end());
+
+ filter_result.count = result_ids.size();
+ filter_result.docs = new uint32_t[result_ids.size()];
+ std::copy(result_ids.begin(), result_ids.end(), filter_result.docs);
+ }
+
+ if (!index->field_is_indexed(a_filter.field_name)) {
+ is_valid = false;
+ return;
+ }
+
+ field f = index->search_schema.at(a_filter.field_name);
+
+ if (f.is_string()) {
+ art_tree* t = index->search_index.at(a_filter.field_name);
+
+ for (const std::string& filter_value : a_filter.values) {
+ std::vector<void*> posting_lists;
+
+ // there could be multiple tokens in a filter value, which we have to treat as ANDs
+ // e.g. country: South Africa
+ Tokenizer tokenizer(filter_value, true, false, f.locale, index->symbols_to_index, index->token_separators);
+
+ std::string str_token;
+ size_t token_index = 0;
+ std::vector<std::string> str_tokens;
+
+ while (tokenizer.next(str_token, token_index)) {
+ str_tokens.push_back(str_token);
+
+ art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
+ str_token.length()+1);
+ if (leaf == nullptr) {
+ continue;
+ }
+
+ posting_lists.push_back(leaf->values);
+ }
+
+ if (posting_lists.size() != str_tokens.size()) {
+ continue;
+ }
+
+ std::vector<posting_list_t*> plists;
+ posting_t::to_expanded_plists(posting_lists, plists, expanded_plists);
+
+ posting_list_iterators.emplace_back(std::vector<posting_list_t::iterator_t>());
+
+ for (auto const& plist: plists) {
+ posting_list_iterators.back().push_back(plist->new_iterator());
+ }
+ }
+
+ doc_matching_string_filter(f.is_array());
+ return;
+ }
+}
+
+bool filter_result_iterator_t::valid() {
+ if (!is_valid) {
+ return false;
+ }
+
+ if (filter_node->isOperator) {
+ if (filter_node->filter_operator == AND) {
+ is_valid = left_it->valid() && right_it->valid();
+ return is_valid;
+ } else {
+ is_valid = left_it->valid() || right_it->valid();
+ return is_valid;
+ }
+ }
+
+ const filter a_filter = filter_node->filter_exp;
+
+ if (!a_filter.referenced_collection_name.empty() || a_filter.field_name == "id") {
+ is_valid = result_index < filter_result.count;
+ return is_valid;
+ }
+
+ if (!index->field_is_indexed(a_filter.field_name)) {
+ is_valid = false;
+ return is_valid;
+ }
+
+ field f = index->search_schema.at(a_filter.field_name);
+
+ if (f.is_string()) {
+ bool one_is_valid = false;
+ for (auto& filter_value_tokens: posting_list_iterators) {
+ posting_list_t::intersect(filter_value_tokens, one_is_valid);
+
+ if (one_is_valid) {
+ break;
+ }
+ }
+
+ is_valid = one_is_valid;
+ return is_valid;
+ }
+
+ return false;
+}
+
+void filter_result_iterator_t::skip_to(uint32_t id) {
+ if (!is_valid) {
+ return;
+ }
+
+ if (filter_node->isOperator) {
+ // Skip the subtrees to id and then apply operators to arrive at the next valid doc.
+ left_it->skip_to(id);
+ right_it->skip_to(id);
+
+ if (filter_node->filter_operator == AND) {
+ and_filter_iterators();
+ } else {
+ or_filter_iterators();
+ }
+
+ return;
+ }
+
+ const filter a_filter = filter_node->filter_exp;
+
+ bool is_referenced_filter = !a_filter.referenced_collection_name.empty();
+ if (is_referenced_filter) {
+ while (filter_result.docs[result_index] < id && ++result_index < filter_result.count);
+
+ if (result_index >= filter_result.count) {
+ is_valid = false;
+ return;
+ }
+
+ seq_id = filter_result.docs[result_index];
+ reference.clear();
+ for (auto const& item: filter_result.reference_filter_results) {
+ reference[item.first] = item.second[result_index];
+ }
+
+ return;
+ }
+
+ if (a_filter.field_name == "id") {
+ while (filter_result.docs[result_index] < id && ++result_index < filter_result.count);
+
+ if (result_index >= filter_result.count) {
+ is_valid = false;
+ return;
+ }
+
+ seq_id = filter_result.docs[result_index];
+ return;
+ }
+
+ if (!index->field_is_indexed(a_filter.field_name)) {
+ is_valid = false;
+ return;
+ }
+
+ field f = index->search_schema.at(a_filter.field_name);
+
+ if (f.is_string()) {
+ // Skip all the token iterators and find a new match.
+ for (auto& filter_value_tokens : posting_list_iterators) {
+ for (auto& token: filter_value_tokens) {
+ // We perform AND on tokens. Short-circuiting here.
+ if (!token.valid()) {
+ break;
+ }
+
+ token.skip_to(id);
+ }
+ }
+
+ doc_matching_string_filter(f.is_array());
+ return;
+ }
+}
+
+int filter_result_iterator_t::valid(uint32_t id) {
+ if (!is_valid) {
+ return -1;
+ }
+
+ if (filter_node->isOperator) {
+ auto left_valid = left_it->valid(id), right_valid = right_it->valid(id);
+
+ if (filter_node->filter_operator == AND) {
+ is_valid = left_it->is_valid && right_it->is_valid;
+
+ if (left_valid < 1 || right_valid < 1) {
+ if (left_valid == -1 || right_valid == -1) {
+ return -1;
+ }
+
+ return 0;
+ }
+
+ return 1;
+ } else {
+ is_valid = left_it->is_valid || right_it->is_valid;
+
+ if (left_valid < 1 && right_valid < 1) {
+ if (left_valid == -1 && right_valid == -1) {
+ return -1;
+ }
+
+ return 0;
+ }
+
+ return 1;
+ }
+ }
+
+ if (filter_node->filter_exp.apply_not_equals) {
+ // Even when iterator becomes invalid, we keep it marked as valid since we are evaluating not equals.
+ if (!valid()) {
+ is_valid = true;
+ return 1;
+ }
+
+ skip_to(id);
+
+ if (!is_valid) {
+ is_valid = true;
+ return 1;
+ }
+
+ return seq_id != id ? 1 : 0;
+ }
+
+ skip_to(id);
+ return is_valid ? (seq_id == id ? 1 : 0) : -1;
+}
+
+Option<bool> filter_result_iterator_t::init_status() {
+ if (filter_node != nullptr && filter_node->isOperator) {
+ auto left_status = left_it->init_status();
+
+ return !left_status.ok() ? left_status : right_it->init_status();
+ }
+
+ return status;
+}
+
+bool filter_result_iterator_t::contains_atleast_one(const void *obj) {
+ if(IS_COMPACT_POSTING(obj)) {
+ compact_posting_list_t* list = COMPACT_POSTING_PTR(obj);
+
+ size_t i = 0;
+ while(i < list->length && valid()) {
+ size_t num_existing_offsets = list->id_offsets[i];
+ size_t existing_id = list->id_offsets[i + num_existing_offsets + 1];
+
+ if (existing_id == seq_id) {
+ return true;
+ }
+
+ // advance smallest value
+ if (existing_id < seq_id) {
+ i += num_existing_offsets + 2;
+ } else {
+ skip_to(existing_id);
+ }
+ }
+ } else {
+ auto list = (posting_list_t*)(obj);
+ posting_list_t::iterator_t it = list->new_iterator();
+
+ while(it.valid() && valid()) {
+ uint32_t id = it.id();
+
+ if(id == seq_id) {
+ return true;
+ }
+
+ if(id < seq_id) {
+ it.skip_to(seq_id);
+ } else {
+ skip_to(id);
+ }
+ }
+ }
+
+ return false;
+}
+
+void filter_result_iterator_t::reset() {
+ if (filter_node == nullptr) {
+ return;
+ }
+
+ if (filter_node->isOperator) {
+ // Reset the subtrees then apply operators to arrive at the first valid doc.
+ left_it->reset();
+ right_it->reset();
+
+ if (filter_node->filter_operator == AND) {
+ and_filter_iterators();
+ } else {
+ or_filter_iterators();
+ }
+
+ return;
+ }
+
+ const filter a_filter = filter_node->filter_exp;
+
+ bool is_referenced_filter = !a_filter.referenced_collection_name.empty();
+ if (is_referenced_filter || a_filter.field_name == "id") {
+ result_index = 0;
+ is_valid = filter_result.count > 0;
+ return;
+ }
+
+ if (!index->field_is_indexed(a_filter.field_name)) {
+ return;
+ }
+
+ field f = index->search_schema.at(a_filter.field_name);
+
+ if (f.is_string()) {
+ posting_list_iterators.clear();
+ for(auto expanded_plist: expanded_plists) {
+ delete expanded_plist;
+ }
+ expanded_plists.clear();
+
+ init();
+ return;
+ }
+}
+
+uint32_t filter_result_iterator_t::to_filter_id_array(uint32_t*& filter_array) {
+ if (!valid()) {
+ return 0;
+ }
+
+ std::vector<uint32_t> filter_ids;
+ do {
+ filter_ids.push_back(seq_id);
+ next();
+ } while (valid());
+
+ filter_array = new uint32_t[filter_ids.size()];
+ std::copy(filter_ids.begin(), filter_ids.end(), filter_array);
+
+ return filter_ids.size();
+}
+
+uint32_t filter_result_iterator_t::and_scalar(const uint32_t* A, const uint32_t& lenA, uint32_t*& results) {
+ if (!valid()) {
+ return 0;
+ }
+
+ std::vector<uint32_t> filter_ids;
+ for (uint32_t i = 0; i < lenA; i++) {
+ auto result = valid(A[i]);
+
+ if (result == -1) {
+ break;
+ }
+
+ if (result == 1) {
+ filter_ids.push_back(A[i]);
+ }
+ }
+
+ if (filter_ids.empty()) {
+ return 0;
+ }
+
+ results = new uint32_t[filter_ids.size()];
+ std::copy(filter_ids.begin(), filter_ids.end(), results);
+
+ return filter_ids.size();
+}
+
+filter_result_iterator_t::filter_result_iterator_t(const std::string collection_name, const Index *const index,
+ const filter_node_t *const filter_node) :
+ collection_name(collection_name),
+ index(index),
+ filter_node(filter_node) {
+ if (filter_node == nullptr) {
+ is_valid = false;
+ return;
+ }
+
+ // Generate the iterator tree and then initialize each node.
+ if (filter_node->isOperator) {
+ left_it = new filter_result_iterator_t(collection_name, index, filter_node->left);
+ right_it = new filter_result_iterator_t(collection_name, index, filter_node->right);
+ }
+
+ init();
+}
+
+filter_result_iterator_t::~filter_result_iterator_t() {
+ // In case the filter was on string field.
+ for(auto expanded_plist: expanded_plists) {
+ delete expanded_plist;
+ }
+
+ delete left_it;
+ delete right_it;
+}
+
+filter_result_iterator_t &filter_result_iterator_t::operator=(filter_result_iterator_t &&obj) noexcept {
+ if (&obj == this)
+ return *this;
+
+ // In case the filter was on string field.
+ for(auto expanded_plist: expanded_plists) {
+ delete expanded_plist;
+ }
+
+ delete left_it;
+ delete right_it;
+
+ collection_name = obj.collection_name;
+ index = obj.index;
+ filter_node = obj.filter_node;
+ left_it = obj.left_it;
+ right_it = obj.right_it;
+
+ obj.left_it = nullptr;
+ obj.right_it = nullptr;
+
+ result_index = obj.result_index;
+
+ filter_result = std::move(obj.filter_result);
+
+ posting_list_iterators = std::move(obj.posting_list_iterators);
+ expanded_plists = std::move(obj.expanded_plists);
+
+ is_valid = obj.is_valid;
+
+ seq_id = obj.seq_id;
+ reference = std::move(obj.reference);
+ status = std::move(obj.status);
+
+ return *this;
+}
diff --git a/src/index.cpp b/src/index.cpp
index 80c0ab9e..c076f8ed 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -5,10 +5,10 @@
#include <set>
#include <unordered_map>
#include <random>
+#include <art.h>
#include <array_utils.h>
#include <match_score.h>
#include <string_utils.h>
-#include <art.h>
#include <tokenizer.h>
#include <s2/s2point.h>
#include <s2/s2latlng.h>
@@ -24,6 +24,7 @@
#include <timsort.hpp>
#include "logger.h"
#include <collection_manager.h>
+#include "validator.h"
#define RETURN_CIRCUIT_BREAKER if((std::chrono::duration_cast<std::chrono::microseconds>( \
std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) { \
@@ -1285,7 +1286,7 @@ void Index::aggregate_topster(Topster* agg_topster, Topster* index_topster) {
void Index::search_all_candidates(const size_t num_search_fields,
const text_match_type_t match_type,
const std::vector<search_field_t>& the_fields,
- const uint32_t* filter_ids, size_t filter_ids_length,
+ filter_result_iterator_t& filter_result_iterator,
const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
const std::unordered_set<uint32_t>& excluded_group_ids,
const std::vector<sort_by>& sort_fields,
@@ -1354,7 +1355,8 @@ void Index::search_all_candidates(const size_t num_search_fields,
sort_fields, topster,groups_processed,
searched_queries, qtoken_set, dropped_tokens, group_limit, group_by_fields,
prioritize_exact_match, prioritize_token_position,
- filter_ids, filter_ids_length, total_cost, syn_orig_num_tokens,
+ filter_result_iterator,
+ total_cost, syn_orig_num_tokens,
exclude_token_ids, exclude_token_ids_size, excluded_group_ids,
sort_order, field_values, geopoint_indices,
id_buff, all_result_ids, all_result_ids_len);
@@ -2728,19 +2730,19 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
const std::string& collection_name) const {
std::shared_lock lock(mutex);
- uint32_t filter_ids_length = 0;
- auto rearrange_op = rearrange_filter_tree(filter_tree_root, filter_ids_length, collection_name);
+ uint32_t approx_filter_ids_length = 0;
+ auto rearrange_op = rearrange_filter_tree(filter_tree_root, approx_filter_ids_length, collection_name);
if (!rearrange_op.ok()) {
return rearrange_op;
}
- filter_result_t filter_result;
- auto filter_op = recursive_filter(filter_tree_root, filter_result, collection_name);
- if (!filter_op.ok()) {
- return filter_op;
+ auto filter_result_iterator = filter_result_iterator_t(collection_name, this, filter_tree_root);
+ auto filter_init_op = filter_result_iterator.init_status();
+ if (!filter_init_op.ok()) {
+ return filter_init_op;
}
- if (filter_tree_root != nullptr && filter_result.count == 0) {
+ if (filter_tree_root != nullptr && !filter_result_iterator.valid()) {
return Option(true);
}
@@ -2752,8 +2754,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
std::unordered_set<uint32_t> excluded_group_ids;
process_curated_ids(included_ids, excluded_ids, group_by_fields, group_limit, filter_curated_hits,
- filter_result.docs, filter_result.count, curated_ids, included_ids_map,
+ filter_result_iterator, curated_ids, included_ids_map,
included_ids_vec, excluded_group_ids);
+ filter_result_iterator.reset();
std::vector<uint32_t> curated_ids_sorted(curated_ids.begin(), curated_ids.end());
std::sort(curated_ids_sorted.begin(), curated_ids_sorted.end());
@@ -2784,7 +2787,10 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
field_query_tokens[0].q_include_tokens[0].value == "*";
+ // TODO: Do AND with phrase ids at last
// handle phrase searches
+ uint32_t* phrase_result_ids = nullptr;
+ uint32_t phrase_result_count = 0;
if (!field_query_tokens[0].q_phrases.empty()) {
do_phrase_search(num_search_fields, the_fields, field_query_tokens,
sort_fields_std, searched_queries, group_limit, group_by_fields,
@@ -2792,8 +2798,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
all_result_ids, all_result_ids_len, groups_processed, curated_ids,
excluded_result_ids, excluded_result_ids_size, excluded_group_ids, curated_topster,
included_ids_map, is_wildcard_query,
- filter_result.docs, filter_result.count);
- if (filter_result.count == 0) {
+ phrase_result_ids, phrase_result_count);
+ if (phrase_result_count == 0) {
goto process_search_results;
}
}
@@ -2801,7 +2807,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
// for phrase query, parser will set field_query_tokens to "*", need to handle that
if (is_wildcard_query && field_query_tokens[0].q_phrases.empty()) {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
- bool no_filters_provided = (filter_tree_root == nullptr && filter_result.count == 0);
+ bool no_filters_provided = (filter_tree_root == nullptr && !filter_result_iterator.valid());
if(no_filters_provided && facets.empty() && curated_ids.empty() && vector_query.field_name.empty() &&
sort_fields_std.size() == 1 && sort_fields_std[0].name == sort_field_const::seq_id &&
@@ -2843,15 +2849,18 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
goto process_search_results;
}
- // if filters were not provided, use the seq_ids index to generate the
- // list of all document ids
+ // if filters were not provided, use the seq_ids index to generate the list of all document ids
if (no_filters_provided) {
- filter_result.count = seq_ids->num_ids();
- filter_result.docs = seq_ids->uncompress();
+ const std::string doc_id_prefix = std::to_string(collection_id) + "_" + Collection::DOC_ID_PREFIX + "_";
+ Option<bool> parse_filter_op = filter::parse_filter_query(SEQ_IDS_FILTER, search_schema,
+ store, doc_id_prefix, filter_tree_root);
+
+ filter_result_iterator = filter_result_iterator_t(collection_name, this, filter_tree_root);
}
- curate_filtered_ids(curated_ids, excluded_result_ids,
- excluded_result_ids_size, filter_result.docs, filter_result.count, curated_ids_sorted);
+// TODO: Curate ids at last
+// curate_filtered_ids(curated_ids, excluded_result_ids,
+// excluded_result_ids_size, filter_result.docs, filter_result.count, curated_ids_sorted);
collate_included_ids({}, included_ids_map, curated_topster, searched_queries);
if (!vector_query.field_name.empty()) {
@@ -2861,37 +2870,46 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
k++;
}
- VectorFilterFunctor filterFunctor(filter_result.docs, filter_result.count);
auto& field_vector_index = vector_index.at(vector_query.field_name);
std::vector<std::pair<float, size_t>> dist_labels;
- if(!no_filters_provided && filter_result.count < vector_query.flat_search_cutoff) {
- for(size_t i = 0; i < filter_result.count; i++) {
- auto seq_id = filter_result.docs[i];
- std::vector<float> values;
+ uint32_t filter_id_count = 0;
+ while (!no_filters_provided &&
+ filter_id_count < vector_query.flat_search_cutoff &&
+ filter_result_iterator.valid()) {
+ auto seq_id = filter_result_iterator.seq_id;
+ std::vector<float> values;
+
+ try {
+ values = field_vector_index->vecdex->getDataByLabel<float>(seq_id);
+ } catch(...) {
+ // likely not found
+ continue;
+ }
- try {
- values = field_vector_index->vecdex->getDataByLabel<float>(seq_id);
- } catch(...) {
- // likely not found
- continue;
- }
+ float dist;
+ if(field_vector_index->distance_type == cosine) {
+ std::vector<float> normalized_q(vector_query.values.size());
+ hnsw_index_t::normalize_vector(vector_query.values, normalized_q);
+ dist = field_vector_index->space->get_dist_func()(normalized_q.data(), values.data(),
+ &field_vector_index->num_dim);
+ } else {
+ dist = field_vector_index->space->get_dist_func()(vector_query.values.data(), values.data(),
+ &field_vector_index->num_dim);
+ }
- float dist;
- if(field_vector_index->distance_type == cosine) {
- std::vector<float> normalized_q(vector_query.values.size());
- hnsw_index_t::normalize_vector(vector_query.values, normalized_q);
- dist = field_vector_index->space->get_dist_func()(normalized_q.data(), values.data(),
- &field_vector_index->num_dim);
- } else {
- dist = field_vector_index->space->get_dist_func()(vector_query.values.data(), values.data(),
- &field_vector_index->num_dim);
- }
+ dist_labels.emplace_back(dist, seq_id);
+ filter_result_iterator.next();
+ filter_id_count++;
+ }
+
+ if(no_filters_provided ||
+ (filter_id_count >= vector_query.flat_search_cutoff && filter_result_iterator.valid())) {
+ dist_labels.clear();
+
+ VectorFilterFunctor filterFunctor(&filter_result_iterator);
- dist_labels.emplace_back(dist, seq_id);
- }
- } else {
if(field_vector_index->distance_type == cosine) {
std::vector<float> normalized_q(vector_query.values.size());
hnsw_index_t::normalize_vector(vector_query.values, normalized_q);
@@ -2901,6 +2919,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
}
+ filter_result_iterator.reset();
+
std::vector<uint32_t> nearest_ids;
for (const auto& dist_label : dist_labels) {
@@ -2952,7 +2972,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
curated_ids, curated_ids_sorted,
excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
all_result_ids, all_result_ids_len,
- filter_result.docs, filter_result.count, concurrency,
+ filter_result_iterator, approx_filter_ids_length, concurrency,
sort_order, field_values, geopoint_indices);
}
} else {
@@ -2994,7 +3014,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
fuzzy_search_fields(the_fields, field_query_tokens[0].q_include_tokens, {}, match_type, excluded_result_ids,
- excluded_result_ids_size, filter_result.docs, filter_result.count, curated_ids_sorted,
+ excluded_result_ids_size, filter_result_iterator, curated_ids_sorted,
excluded_group_ids,
sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,
all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
@@ -3002,6 +3022,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
typo_tokens_threshold, exhaustive_search,
max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order,
field_values, geopoint_indices);
+ filter_result_iterator.reset();
// try split/joining tokens if no results are found
if(split_join_tokens == always || (all_result_ids_len == 0 && split_join_tokens == fallback)) {
@@ -3032,12 +3053,13 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
fuzzy_search_fields(the_fields, resolved_tokens, {}, match_type, excluded_result_ids,
- excluded_result_ids_size, filter_result.docs, filter_result.count, curated_ids_sorted,
+ excluded_result_ids_size, filter_result_iterator, curated_ids_sorted,
excluded_group_ids,
sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,
all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
prioritize_token_position, query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search,
max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order, field_values, geopoint_indices);
+ filter_result_iterator.reset();
}
}
@@ -3050,9 +3072,10 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
topster, q_pos_synonyms, syn_orig_num_tokens,
groups_processed, searched_queries, all_result_ids, all_result_ids_len,
- filter_result.docs, filter_result.count, query_hashes,
+ filter_result_iterator, query_hashes,
sort_order, field_values, geopoint_indices,
qtoken_set);
+ filter_result_iterator.reset();
// gather up both original query and synonym queries and do drop tokens
@@ -3101,7 +3124,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
fuzzy_search_fields(the_fields, truncated_tokens, dropped_tokens, match_type,
excluded_result_ids, excluded_result_ids_size,
- filter_result.docs, filter_result.count,
+ filter_result_iterator,
curated_ids_sorted, excluded_group_ids, sort_fields_std, num_typos, searched_queries,
qtoken_set, topster, groups_processed,
all_result_ids, all_result_ids_len, group_limit, group_by_fields,
@@ -3109,6 +3132,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
token_order, prefixes, typo_tokens_threshold,
exhaustive_search, max_candidates, min_len_1typo,
min_len_2typo, -1, sort_order, field_values, geopoint_indices);
+ filter_result_iterator.reset();
} else {
break;
@@ -3121,10 +3145,12 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
group_limit, group_by_fields,
max_extra_prefix, max_extra_suffix,
field_query_tokens[0].q_include_tokens,
- topster, filter_result.docs, filter_result.count,
+ topster, filter_result_iterator,
sort_order, field_values, geopoint_indices,
curated_ids_sorted, excluded_group_ids, all_result_ids, all_result_ids_len, groups_processed);
+ filter_result_iterator.reset();
+
if(!vector_query.field_name.empty()) {
// check at least one of sort fields is text match
bool has_text_match = false;
@@ -3140,7 +3166,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
constexpr float TEXT_MATCH_WEIGHT = 0.7;
constexpr float VECTOR_SEARCH_WEIGHT = 1.0 - TEXT_MATCH_WEIGHT;
- VectorFilterFunctor filterFunctor(filter_result.docs, filter_result.count);
+ VectorFilterFunctor filterFunctor(&filter_result_iterator);
auto& field_vector_index = vector_index.at(vector_query.field_name);
std::vector<std::pair<float, size_t>> dist_labels;
auto k = std::max<size_t>(vector_query.k, fetch_size);
@@ -3152,6 +3178,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
} else {
dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, &filterFunctor);
}
+ filter_result_iterator.reset();
std::vector<std::pair<uint32_t,float>> vec_results;
for (const auto& dist_label : dist_labels) {
@@ -3361,7 +3388,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
void Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
const std::vector<uint32_t>& excluded_ids,
const std::vector<std::string>& group_by_fields, const size_t group_limit,
- const bool filter_curated_hits, const uint32_t* filter_ids, uint32_t filter_ids_length,
+ const bool filter_curated_hits, filter_result_iterator_t& filter_result_iterator,
std::set<uint32_t>& curated_ids,
std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
std::vector<uint32_t>& included_ids_vec,
@@ -3384,19 +3411,18 @@ void Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>
// if `filter_curated_hits` is enabled, we will remove curated hits that don't match filter condition
std::set<uint32_t> included_ids_set;
- if(filter_ids_length != 0 && filter_curated_hits) {
- uint32_t* included_ids_arr = nullptr;
- size_t included_ids_len = ArrayUtils::and_scalar(&included_ids_vec[0], included_ids_vec.size(), filter_ids,
- filter_ids_length, &included_ids_arr);
+ if(filter_result_iterator.valid() && filter_curated_hits) {
+ for (const auto &included_id: included_ids_vec) {
+ auto result = filter_result_iterator.valid(included_id);
- included_ids_vec.clear();
+ if (result == -1) {
+ break;
+ }
- for(size_t i = 0; i < included_ids_len; i++) {
- included_ids_set.insert(included_ids_arr[i]);
- included_ids_vec.push_back(included_ids_arr[i]);
+ if (result == 1) {
+ included_ids_set.insert(included_id);
+ }
}
-
- delete [] included_ids_arr;
} else {
included_ids_set.insert(included_ids_vec.begin(), included_ids_vec.end());
}
@@ -3454,7 +3480,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
const text_match_type_t match_type,
const uint32_t* exclude_token_ids,
size_t exclude_token_ids_size,
- const uint32_t* filter_ids, size_t filter_ids_length,
+ filter_result_iterator_t& filter_result_iterator,
const std::vector<uint32_t>& curated_ids,
const std::unordered_set<uint32_t>& excluded_group_ids,
const std::vector<sort_by> & sort_fields,
@@ -3597,9 +3623,10 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
const auto& prev_token = last_token ? token_candidates_vec.back().candidates[0] : "";
std::vector<art_leaf*> field_leaves;
- art_fuzzy_search(search_index.at(the_field.name), (const unsigned char *) token.c_str(), token_len,
+ art_fuzzy_search_i(search_index.at(the_field.name), (const unsigned char *) token.c_str(), token_len,
costs[token_index], costs[token_index], max_candidates, token_order, prefix_search,
- last_token, prev_token, filter_ids, filter_ids_length, field_leaves, unique_tokens);
+ last_token, prev_token, filter_result_iterator, field_leaves, unique_tokens);
+ filter_result_iterator.reset();
/*auto timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
@@ -3628,8 +3655,9 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
std::vector<uint32_t> prev_token_doc_ids;
find_across_fields(token_candidates_vec.back().token,
token_candidates_vec.back().candidates[0],
- the_fields, num_search_fields, filter_ids, filter_ids_length, exclude_token_ids,
+ the_fields, num_search_fields, filter_result_iterator, exclude_token_ids,
exclude_token_ids_size, prev_token_doc_ids, popular_field_ids);
+ filter_result_iterator.reset();
for(size_t field_id: query_field_ids) {
auto& the_field = the_fields[field_id];
@@ -3649,9 +3677,9 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
}
std::vector<art_leaf*> field_leaves;
- art_fuzzy_search(search_index.at(the_field.name), (const unsigned char *) token.c_str(), token_len,
+ art_fuzzy_search_i(search_index.at(the_field.name), (const unsigned char *) token.c_str(), token_len,
costs[token_index], costs[token_index], max_candidates, token_order, prefix_search,
- false, "", filter_ids, filter_ids_length, field_leaves, unique_tokens);
+ false, "", filter_result_iterator, field_leaves, unique_tokens);
if(field_leaves.empty()) {
// look at the next field
@@ -3704,7 +3732,8 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
if(token_candidates_vec.size() == query_tokens.size()) {
std::vector<uint32_t> id_buff;
- search_all_candidates(num_search_fields, match_type, the_fields, filter_ids, filter_ids_length,
+
+ search_all_candidates(num_search_fields, match_type, the_fields, filter_result_iterator,
exclude_token_ids, exclude_token_ids_size, excluded_group_ids,
sort_fields, token_candidates_vec, searched_queries, qtoken_set,
dropped_tokens, topster,
@@ -3714,6 +3743,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
exhaustive_search, max_candidates,
syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
query_hashes, id_buff);
+ filter_result_iterator.reset();
if(id_buff.size() > 1) {
gfx::timsort(id_buff.begin(), id_buff.end());
@@ -3774,7 +3804,7 @@ void Index::find_across_fields(const token_t& previous_token,
const std::string& previous_token_str,
const std::vector<search_field_t>& the_fields,
const size_t num_search_fields,
- const uint32_t* filter_ids, uint32_t filter_ids_length,
+ filter_result_iterator_t& filter_result_iterator,
const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
std::vector<uint32_t>& prev_token_doc_ids,
std::vector<size_t>& top_prefix_field_ids) const {
@@ -3785,7 +3815,7 @@ void Index::find_across_fields(const token_t& previous_token,
// used to track plists that must be destructed once done
std::vector<posting_list_t*> expanded_plists;
- result_iter_state_t istate(exclude_token_ids, exclude_token_ids_size, filter_ids, filter_ids_length);
+ result_iter_state_t istate(exclude_token_ids, exclude_token_ids_size, &filter_result_iterator);
const bool prefix_search = previous_token.is_prefix_searched;
const uint32_t token_num_typos = previous_token.num_typos;
@@ -3866,7 +3896,7 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
const std::vector<std::string>& group_by_fields,
const bool prioritize_exact_match,
const bool prioritize_token_position,
- const uint32_t* filter_ids, uint32_t filter_ids_length,
+ filter_result_iterator_t& filter_result_iterator,
const uint32_t total_cost, const int syn_orig_num_tokens,
const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
const std::unordered_set<uint32_t>& excluded_group_ids,
@@ -3927,7 +3957,7 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
// used to track plists that must be destructed once done
std::vector<posting_list_t*> expanded_plists;
- result_iter_state_t istate(exclude_token_ids, exclude_token_ids_size, filter_ids, filter_ids_length);
+ result_iter_state_t istate(exclude_token_ids, exclude_token_ids_size, &filter_result_iterator);
// for each token, find the posting lists across all query_by fields
for(size_t ti = 0; ti < query_tokens.size(); ti++) {
@@ -4397,13 +4427,10 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<s
Topster* curated_topster,
const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
bool is_wildcard_query,
- uint32_t*& filter_ids, uint32_t& filter_ids_length) const {
+ uint32_t*& phrase_result_ids, uint32_t& phrase_result_count) const {
std::map<uint32_t, size_t> phrase_match_id_scores;
- uint32_t* phrase_match_ids = nullptr;
- size_t phrase_match_ids_size = 0;
-
for(size_t i = 0; i < num_search_fields; i++) {
const std::string& field_name = search_fields[i].name;
const size_t field_weight = search_fields[i].weight;
@@ -4472,50 +4499,30 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<s
}
// across fields, we have to OR phrase match ids
- if(phrase_match_ids_size == 0) {
- phrase_match_ids = field_phrase_match_ids;
- phrase_match_ids_size = field_phrase_match_ids_size;
+ if(phrase_result_count == 0) {
+ phrase_result_ids = field_phrase_match_ids;
+ phrase_result_count = field_phrase_match_ids_size;
} else {
uint32_t* phrase_ids_merged = nullptr;
- phrase_match_ids_size = ArrayUtils::or_scalar(phrase_match_ids, phrase_match_ids_size, field_phrase_match_ids,
+ phrase_result_count = ArrayUtils::or_scalar(phrase_result_ids, phrase_result_count, field_phrase_match_ids,
field_phrase_match_ids_size, &phrase_ids_merged);
- delete [] phrase_match_ids;
+ delete [] phrase_result_ids;
delete [] field_phrase_match_ids;
- phrase_match_ids = phrase_ids_merged;
+ phrase_result_ids = phrase_ids_merged;
}
}
- // AND phrase id matches with filter ids
- if(filter_ids_length == 0) {
- filter_ids = phrase_match_ids;
- filter_ids_length = phrase_match_ids_size;
- } else {
- uint32_t* filter_ids_merged = nullptr;
- filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, phrase_match_ids,
- phrase_match_ids_size, &filter_ids_merged);
-
- delete [] filter_ids;
- filter_ids = filter_ids_merged;
-
- delete [] phrase_match_ids;
- }
-
- if(filter_ids_length == 0) {
- delete [] filter_ids;
- filter_ids = nullptr;
- }
-
curate_filtered_ids(curated_ids, excluded_result_ids,
- excluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted);
+ excluded_result_ids_size, phrase_result_ids, phrase_result_count, curated_ids_sorted);
collate_included_ids({}, included_ids_map, curated_topster, searched_queries);
size_t filter_index = 0;
if(is_wildcard_query) {
- all_result_ids = new uint32_t[filter_ids_length];
- std::copy(filter_ids, filter_ids + filter_ids_length, all_result_ids);
- all_result_ids_len = filter_ids_length;
+ all_result_ids = new uint32_t[phrase_result_count];
+ std::copy(phrase_result_ids, phrase_result_ids + phrase_result_count, all_result_ids);
+ all_result_ids_len = phrase_result_count;
} else {
// this means that the there are non-phrase tokens in the query
// so we cannot directly copy to the all_result_ids array
@@ -4523,8 +4530,8 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<s
}
// populate topster
- for(size_t i = 0; i < std::min<size_t>(10000, filter_ids_length); i++) {
- auto seq_id = filter_ids[i];
+ for(size_t i = 0; i < std::min<size_t>(10000, phrase_result_count); i++) {
+ auto seq_id = phrase_result_ids[i];
int64_t match_score = phrase_match_id_scores[seq_id];
int64_t scores[3] = {0};
@@ -4577,7 +4584,7 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries,
uint32_t*& all_result_ids, size_t& all_result_ids_len,
- const uint32_t* filter_ids, const uint32_t filter_ids_length,
+ filter_result_iterator_t& filter_result_iterator,
std::set<uint64>& query_hashes,
const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
@@ -4587,7 +4594,7 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
for (const auto& syn_tokens : q_pos_synonyms) {
query_hashes.clear();
fuzzy_search_fields(the_fields, syn_tokens, {}, match_type, exclude_token_ids,
- exclude_token_ids_size, filter_ids, filter_ids_length, curated_ids_sorted, excluded_group_ids,
+ exclude_token_ids_size, filter_result_iterator, curated_ids_sorted, excluded_group_ids,
sort_fields_std, {0}, searched_queries, qtoken_set, actual_topster, groups_processed,
all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
prioritize_token_position, query_hashes, token_order, prefixes, typo_tokens_threshold,
@@ -4605,7 +4612,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,
const size_t max_extra_suffix,
const std::vector<token_t>& query_tokens, Topster* actual_topster,
- const uint32_t *filter_ids, size_t filter_ids_length,
+ filter_result_iterator_t& filter_result_iterator,
const int sort_order[3],
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
const std::vector<size_t>& geopoint_indices,
@@ -4639,10 +4646,11 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
raw_infix_ids_length = infix_ids.size();
}
- if(filter_ids_length != 0) {
+ if(filter_result_iterator.valid()) {
uint32_t *filtered_raw_infix_ids = nullptr;
- raw_infix_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, raw_infix_ids,
- raw_infix_ids_length, &filtered_raw_infix_ids);
+
+ raw_infix_ids_length = filter_result_iterator.and_scalar(raw_infix_ids, raw_infix_ids_length,
+ filtered_raw_infix_ids);
if(raw_infix_ids != &infix_ids[0]) {
delete [] raw_infix_ids;
}
@@ -4935,20 +4943,21 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,
const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,
- uint32_t*& all_result_ids, size_t& all_result_ids_len, const uint32_t* filter_ids,
- uint32_t filter_ids_length, const size_t concurrency,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ filter_result_iterator_t& filter_result_iterator, const uint32_t& approx_filter_ids_length,
+ const size_t concurrency,
const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
const std::vector<size_t>& geopoint_indices) const {
uint32_t token_bits = 0;
- const bool check_for_circuit_break = (filter_ids_length > 1000000);
+ const bool check_for_circuit_break = (approx_filter_ids_length > 1000000);
//auto beginF = std::chrono::high_resolution_clock::now();
- const size_t num_threads = std::min<size_t>(concurrency, filter_ids_length);
+ const size_t num_threads = std::min<size_t>(concurrency, approx_filter_ids_length);
const size_t window_size = (num_threads == 0) ? 0 :
- (filter_ids_length + num_threads - 1) / num_threads; // rounds up
+ (approx_filter_ids_length + num_threads - 1) / num_threads; // rounds up
spp::sparse_hash_map<uint64_t, uint64_t> tgroups_processed[num_threads];
Topster* topsters[num_threads];
@@ -4965,14 +4974,15 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
const auto parent_search_stop_ms = search_stop_us;
auto parent_search_cutoff = search_cutoff;
- for(size_t thread_id = 0; thread_id < num_threads && filter_index < filter_ids_length; thread_id++) {
- size_t batch_res_len = window_size;
+ for(size_t thread_id = 0; thread_id < num_threads && filter_result_iterator.valid(); thread_id++) {
+ std::vector<uint32_t> batch_result_ids;
+ batch_result_ids.reserve(window_size);
- if(filter_index + window_size > filter_ids_length) {
- batch_res_len = filter_ids_length - filter_index;
- }
+ do {
+ batch_result_ids.push_back(filter_result_iterator.seq_id);
+ filter_result_iterator.next();
+ } while (batch_result_ids.size() < window_size && filter_result_iterator.valid());
- const uint32_t* batch_result_ids = filter_ids + filter_index;
num_queued++;
searched_queries.push_back({});
@@ -4984,7 +4994,7 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
&group_limit, &group_by_fields, &topsters, &tgroups_processed, &excluded_group_ids,
&sort_order, field_values, &geopoint_indices, &plists,
check_for_circuit_break,
- batch_result_ids, batch_res_len,
+ batch_result_ids,
&num_processed, &m_process, &cv_process]() {
search_begin_us = parent_search_begin;
@@ -4993,7 +5003,7 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
size_t filter_index = 0;
- for(size_t i = 0; i < batch_res_len; i++) {
+ for(size_t i = 0; i < batch_result_ids.size(); i++) {
const uint32_t seq_id = batch_result_ids[i];
int64_t match_score = 0;
@@ -5033,7 +5043,7 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
cv_process.notify_one();
});
- filter_index += batch_res_len;
+ filter_index += batch_result_ids.size();
}
std::unique_lock<std::mutex> lock_process(m_process);
@@ -5045,7 +5055,7 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
//groups_processed.insert(tgroups_processed[thread_id].begin(), tgroups_processed[thread_id].end());
for(const auto& it : tgroups_processed[thread_id]) {
groups_processed[it.first]+= it.second;
- }
+ }
aggregate_topster(topster, topsters[thread_id]);
delete topsters[thread_id];
}
@@ -5054,11 +5064,13 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
std::chrono::high_resolution_clock::now() - beginF).count();
LOG(INFO) << "Time for raw scoring: " << timeMillisF;*/
- uint32_t* new_all_result_ids = nullptr;
- all_result_ids_len = ArrayUtils::or_scalar(all_result_ids, all_result_ids_len, filter_ids,
- filter_ids_length, &new_all_result_ids);
- delete [] all_result_ids;
- all_result_ids = new_all_result_ids;
+// TODO: OR filter ids with all_results_ids
+//
+// uint32_t* new_all_result_ids = nullptr;
+// all_result_ids_len = ArrayUtils::or_scalar(all_result_ids, all_result_ids_len, filter_ids,
+// filter_ids_length, &new_all_result_ids);
+// delete [] all_result_ids;
+// all_result_ids = new_all_result_ids;
}
void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint_indices,
diff --git a/src/or_iterator.cpp b/src/or_iterator.cpp
index 13165370..df4404cb 100644
--- a/src/or_iterator.cpp
+++ b/src/or_iterator.cpp
@@ -1,5 +1,5 @@
#include "or_iterator.h"
-
+#include "filter.h"
bool or_iterator_t::at_end(const std::vector<or_iterator_t>& its) {
// if any iterator is invalid, we stop
@@ -208,6 +208,10 @@ bool or_iterator_t::take_id(result_iter_state_t& istate, uint32_t id, bool& is_e
return false;
}
+ if (istate.fit != nullptr) {
+ return (istate.fit->valid(id) == 1);
+ }
+
return true;
}
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index 39f3ac00..e983e98b 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -1260,6 +1260,146 @@ void posting_list_t::get_exact_matches(std::vector<iterator_t>& its, const bool
num_exact_ids = exact_id_index;
}
+bool posting_list_t::has_exact_match(std::vector<posting_list_t::iterator_t>& posting_list_iterators,
+ const bool field_is_array) {
+ if(posting_list_iterators.size() == 1) {
+ return is_single_token_verbatim_match(posting_list_iterators[0], field_is_array);
+ } else {
+
+ if (!field_is_array) {
+ for (uint32_t i = posting_list_iterators.size() - 1; i >= 0; i--) {
+ posting_list_t::iterator_t& it = posting_list_iterators[i];
+
+ block_t* curr_block = it.block();
+ uint32_t curr_index = it.index();
+
+ if(curr_block == nullptr || curr_index == UINT32_MAX) {
+ return false;
+ }
+
+ uint32_t* offsets = it.offsets;
+
+ uint32_t start_offset_index = it.offset_index[curr_index];
+ uint32_t end_offset_index = (curr_index == curr_block->size() - 1) ?
+ curr_block->offsets.getLength() :
+ it.offset_index[curr_index + 1];
+
+ if(i == posting_list_iterators.size() - 1) {
+ // check if the last query token is the last offset
+ if( offsets[end_offset_index-1] != 0 ||
+ (end_offset_index-2 >= 0 && offsets[end_offset_index-2] != posting_list_iterators.size())) {
+ // not the last token for the document, so skip
+ return false;
+ }
+ }
+
+ // looping handles duplicate query tokens, e.g. "hip hip hurray hurray"
+ while(start_offset_index < end_offset_index) {
+ uint32_t offset = offsets[start_offset_index];
+ start_offset_index++;
+
+ if(offset == (i + 1)) {
+ // we have found a matching index, no need to look further
+ return true;
+ }
+
+ if(offset > (i + 1)) {
+ return false;
+ }
+ }
+ }
+ }
+
+ else {
+ // field is an array
+
+ struct token_index_meta_t {
+ std::bitset<32> token_index;
+ bool has_last_token;
+ };
+
+ std::map<size_t, token_index_meta_t> array_index_to_token_index;
+
+ for(uint32_t i = posting_list_iterators.size() - 1; i >= 0; i--) {
+ posting_list_t::iterator_t& it = posting_list_iterators[i];
+
+ block_t* curr_block = it.block();
+ uint32_t curr_index = it.index();
+
+ if(curr_block == nullptr || curr_index == UINT32_MAX) {
+ return false;
+ }
+
+ uint32_t* offsets = it.offsets;
+ uint32_t start_offset_index = it.offset_index[curr_index];
+ uint32_t end_offset_index = (curr_index == curr_block->size() - 1) ?
+ curr_block->offsets.getLength() :
+ it.offset_index[curr_index + 1];
+
+ int prev_pos = -1;
+ bool has_atleast_one_last_token = false;
+ bool found_matching_index = false;
+ size_t num_matching_index = 0;
+
+ while(start_offset_index < end_offset_index) {
+ int pos = offsets[start_offset_index];
+ start_offset_index++;
+
+ if(pos == prev_pos) { // indicates end of array index
+ size_t array_index = (size_t) offsets[start_offset_index];
+
+ if(start_offset_index+1 < end_offset_index) {
+ size_t next_offset = (size_t) offsets[start_offset_index + 1];
+ if(next_offset == 0 && pos == posting_list_iterators.size()) {
+ // indicates that token is the last token on the doc
+ array_index_to_token_index[array_index].has_last_token = true;
+ has_atleast_one_last_token = true;
+ start_offset_index++;
+ }
+ }
+
+ if(found_matching_index) {
+ array_index_to_token_index[array_index].token_index.set(i + 1);
+ }
+
+ start_offset_index++; // skip current value which is the array index or flag for last index
+ prev_pos = -1;
+ found_matching_index = false;
+ continue;
+ }
+
+ if(pos == (i + 1)) {
+ // we have found a matching index
+ found_matching_index = true;
+ num_matching_index++;
+ }
+
+ prev_pos = pos;
+ }
+
+ // check if the last query token is the last offset of ANY array element
+ if(i == posting_list_iterators.size() - 1 && !has_atleast_one_last_token) {
+ return false;
+ }
+
+ if(num_matching_index == 0) {
+ // not even a single matching index found: can never be an exact match
+ return false;
+ }
+ }
+
+ // iterate array index to token index to check if atleast 1 array position contains all tokens
+ for(auto& kv: array_index_to_token_index) {
+ if(kv.second.token_index.count() == posting_list_iterators.size() && kv.second.has_last_token) {
+ return true;
+ }
+ }
+ }
+ }
+
+ return false;
+}
+
bool posting_list_t::found_token_sequence(const std::vector<token_positions_t>& token_positions,
const size_t token_index, const uint16_t target_pos) {
diff --git a/src/validator.cpp b/src/validator.cpp
index 04d99ab2..0caa1302 100644
--- a/src/validator.cpp
+++ b/src/validator.cpp
@@ -1,4 +1,5 @@
#include "validator.h"
+#include "field.h"
Option<uint32_t> validator_t::coerce_element(const field& a_field, nlohmann::json& document,
nlohmann::json& doc_ele,
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index e2ce9044..930c8176 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -6,6 +6,7 @@
#include <filesystem>
#include <cstdlib>
#include <collection_manager.h>
+#include <validator.h>
#include "collection.h"
#include "text_embedder_manager.h"
#include "http_client.h"
diff --git a/test/filter_test.cpp b/test/filter_test.cpp
index b4f836f1..d125e814 100644
--- a/test/filter_test.cpp
+++ b/test/filter_test.cpp
@@ -125,15 +125,25 @@ TEST_F(FilterTest, FilterTreeIterator) {
filter_tree_root);
ASSERT_TRUE(filter_op.ok());
- auto iter_exact_match_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
- ASSERT_TRUE(iter_exact_match_test.init_status().ok());
+ auto iter_exact_match_1_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_exact_match_1_test.init_status().ok());
for (uint32_t i = 0; i < 5; i++) {
- ASSERT_TRUE(iter_exact_match_test.valid());
- ASSERT_EQ(i, iter_exact_match_test.seq_id);
- iter_exact_match_test.next();
+ ASSERT_TRUE(iter_exact_match_1_test.valid());
+ ASSERT_EQ(i, iter_exact_match_1_test.seq_id);
+ iter_exact_match_1_test.next();
}
- ASSERT_FALSE(iter_exact_match_test.valid());
+ ASSERT_FALSE(iter_exact_match_1_test.valid());
+
+ delete filter_tree_root;
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("tags:= PLATINUM", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_exact_match_2_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_exact_match_2_test.init_status().ok());
+ ASSERT_FALSE(iter_exact_match_2_test.valid());
delete filter_tree_root;
filter_tree_root = nullptr;