Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Speed up numerical filter + fixed edge case with -ve value.

kishorenc6883b4db36d9
diff --git a/include/array_base.h b/include/array_base.h
index d4eae6a8..c53e6b12 100644
--- a/include/array_base.h
+++ b/include/array_base.h
@@ -25,7 +25,7 @@ protected:
}
public:
- array_base(const uint32_t n=2) {
+ explicit array_base(const uint32_t n=2) {
size_bytes = METADATA_OVERHEAD + (n * FOR_ELE_SIZE);
in = (uint8_t *) malloc(size_bytes * sizeof *in);
memset(in, 0, size_bytes);
diff --git a/include/index.h b/include/index.h
index 33913276..a0d12e7f 100644
--- a/include/index.h
+++ b/include/index.h
@@ -15,6 +15,7 @@
#include <option.h>
#include <set>
#include "string_utils.h"
+#include "num_tree.h"
struct token_candidates {
std::string token;
@@ -135,6 +136,8 @@ private:
spp::sparse_hash_map<std::string, art_tree*> search_index;
+ spp::sparse_hash_map<std::string, num_tree_t*> numerical_index;
+
// seq_id => (facet => values)
spp::sparse_hash_map<uint32_t, std::vector<std::vector<uint64_t>>> facet_index_v2;
@@ -189,27 +192,9 @@ private:
void index_string_array_field(const std::vector<std::string> & strings, const int64_t score, art_tree *t,
uint32_t seq_id, int facet_id, const field & a_field);
- void index_int32_field(const int32_t value, const int64_t score, art_tree *t, uint32_t seq_id) const;
-
- void index_int64_field(const int64_t value, const int64_t score, art_tree *t, uint32_t seq_id) const;
-
- void index_float_field(const float value, const int64_t score, art_tree *t, uint32_t seq_id) const;
-
- void index_bool_field(const bool value, const int64_t score, art_tree *t, uint32_t seq_id) const;
-
- void index_int32_array_field(const std::vector<int32_t> & values, const int64_t score, art_tree *t, uint32_t seq_id) const;
-
- void index_int64_array_field(const std::vector<int64_t> & values, const int64_t score, art_tree *t, uint32_t seq_id) const;
-
- void index_float_array_field(const std::vector<float> & values, const int64_t score, art_tree *t, uint32_t seq_id) const;
-
- void index_bool_array_field(const std::vector<bool> & values, const int64_t score, art_tree *t, uint32_t seq_id) const;
-
void remove_and_shift_offset_index(sorted_array& offset_index, const uint32_t* indices_sorted,
const uint32_t indices_length);
- uint32_t* collate_leaf_ids(const std::vector<const art_leaf *> &leaves, size_t& result_ids_len) const;
-
void collate_included_ids(const std::vector<std::string>& q_included_tokens,
const std::string & field, const uint8_t field_id,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
@@ -289,6 +274,8 @@ public:
const spp::sparse_hash_map<std::string, art_tree *> &_get_search_index() const;
+ const spp::sparse_hash_map<std::string, num_tree_t*>& _get_numerical_index() const;
+
// for limiting number of results on multiple candidates / query rewrites
enum {TYPO_TOKENS_THRESHOLD = 100};
@@ -328,5 +315,22 @@ public:
void scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, nlohmann::json& old_doc);
void tokenize_doc_field(const nlohmann::json& document, const field& search_field, std::vector<std::string>& tokens);
+
+ template<typename T>
+ bool _arrays_match(std::vector<T> reindex_vals, std::vector<T> old_vals) {
+ if(old_vals.size() != reindex_vals.size()) {
+ return false;
+ }
+
+ for(size_t i=0; i < reindex_vals.size(); i++) {
+ const T& reindex_val = reindex_vals[i];
+ const T& old_val = old_vals[i];
+ if(reindex_val != old_val) {
+ return false;
+ }
+ }
+
+ return true;
+ }
};
diff --git a/include/num_tree.h b/include/num_tree.h
new file mode 100644
index 00000000..6f99f822
--- /dev/null
+++ b/include/num_tree.h
@@ -0,0 +1,121 @@
+#pragma once
+
+#include <map>
+#include "sparsepp.h"
+#include "sorted_array.h"
+#include "array_utils.h"
+
+class num_tree_t {
+private:
+ std::map<int64_t, sorted_array*> int64map;
+
+public:
+
+ ~num_tree_t() {
+ for(auto& kv: int64map) {
+ delete kv.second;
+ }
+ }
+
+ void insert(int64_t value, uint32_t id) {
+ if(int64map.count(value) == 0) {
+ int64map.emplace(value, new sorted_array);
+ }
+
+ int64map[value]->append(id);
+ }
+
+ void search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids, size_t& ids_len) {
+ if(int64map.empty()) {
+ return ;
+ }
+
+ if(comparator == EQUALS) {
+ const auto& it = int64map.find(value);
+ if(it != int64map.end()) {
+ uint32_t *out = nullptr;
+ uint32_t* val_ids = it->second->uncompress();
+ ids_len = ArrayUtils::or_scalar(val_ids, it->second->getLength(),
+ *ids, ids_len, &out);
+ delete[] *ids;
+ *ids = out;
+ delete[] val_ids;
+ }
+ } else if(comparator == GREATER_THAN || comparator == GREATER_THAN_EQUALS) {
+ auto it = int64map.lower_bound(value); // iter values will be >= value
+
+ if(it == int64map.end()) {
+ return ;
+ }
+
+ if(comparator == GREATER_THAN && it->first == value) {
+ it++;
+ }
+
+ while(it != int64map.end()) {
+ uint32_t *out = nullptr;
+ uint32_t *val_ids = it->second->uncompress();
+ ids_len = ArrayUtils::or_scalar(val_ids, it->second->getLength(),
+ *ids, ids_len, &out);
+ delete [] val_ids;
+ delete [] *ids;
+ *ids = out;
+ it++;
+ }
+ } else if(comparator == LESS_THAN || comparator == LESS_THAN_EQUALS) {
+ auto max_iter = int64map.lower_bound(value); // iter values will be >= value
+
+ if(comparator == LESS_THAN) {
+ if(max_iter == int64map.end()) {
+ max_iter--;
+ }
+
+ else if(max_iter == int64map.begin() && max_iter->first != value) {
+ return ;
+ }
+
+ else if(max_iter != int64map.begin() && max_iter->first >= value) {
+ max_iter--;
+ }
+ } else {
+ if(max_iter == int64map.end()) {
+ max_iter--;
+ }
+ }
+
+ auto iter = int64map.begin();
+
+ while(true) {
+ uint32_t* out = nullptr;
+ uint32_t* val_ids = iter->second->uncompress();
+ ids_len = ArrayUtils::or_scalar(val_ids, iter->second->getLength(),
+ *ids, ids_len, &out);
+ delete[] val_ids;
+ delete[] *ids;
+ *ids = out;
+
+ if(iter == max_iter) {
+ break;
+ }
+
+ iter++;
+ }
+ }
+ }
+
+ void remove(uint64_t value, uint32_t id) {
+ if(int64map.count(value) != 0) {
+ sorted_array* arr = int64map[value];
+ arr->remove_value(id);
+ if(arr->getLength() == 0) {
+ delete arr;
+ int64map.erase(value);
+ }
+ }
+ }
+
+ size_t size() {
+ return int64map.size();
+ }
+
+};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index cfe5d908..2d7c9873 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1796,6 +1796,7 @@ Option<bool> Collection::parse_filter_query(const std::string& simple_filter_que
if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
std::vector<std::string> filter_values;
StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+ f = {field_name, {}, {}};
for(std::string & filter_value: filter_values) {
if(filter_value != "true" && filter_value != "false") {
@@ -1804,9 +1805,9 @@ Option<bool> Collection::parse_filter_query(const std::string& simple_filter_que
}
filter_value = (filter_value == "true") ? "1" : "0";
+ f.values.push_back(filter_value);
+ f.comparators.push_back(EQUALS);
}
-
- f = {field_name, filter_values, {EQUALS}};
} else {
if(raw_value != "true" && raw_value != "false") {
return Option<bool>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");
diff --git a/src/index.cpp b/src/index.cpp
index f1726a23..7db552ad 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -15,16 +15,21 @@ Index::Index(const std::string name, const std::unordered_map<std::string, field
std::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema):
name(name), search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema) {
- for(const auto & pair: search_schema) {
- art_tree *t = new art_tree;
- art_tree_init(t);
- search_index.emplace(pair.first, t);
+ for(const auto & fname_field: search_schema) {
+ if(fname_field.second.is_string()) {
+ art_tree *t = new art_tree;
+ art_tree_init(t);
+ search_index.emplace(fname_field.first, t);
+ } else {
+ num_tree_t* num_tree = new num_tree_t;
+ numerical_index.emplace(fname_field.first, num_tree);
+ }
// initialize for non-string facet fields
- if(pair.second.facet && !pair.second.is_string()) {
+ if(fname_field.second.facet && !fname_field.second.is_string()) {
art_tree *ft = new art_tree;
art_tree_init(ft);
- search_index.emplace(pair.second.faceted_name(), ft);
+ search_index.emplace(fname_field.second.faceted_name(), ft);
}
}
@@ -47,6 +52,11 @@ Index::~Index() {
name_tree.second = nullptr;
}
+ for(auto & name_tree: numerical_index) {
+ delete name_tree.second;
+ name_tree.second = nullptr;
+ }
+
search_index.clear();
for(auto & name_map: sort_index) {
@@ -164,38 +174,56 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
}
}
- art_tree *t = search_index.at(field_name);
-
if(field_pair.second.type == field_types::STRING) {
+ art_tree *t = search_index.at(field_name);
const std::string & text = document[field_name];
index_string_field(text, points, t, seq_id, facet_id, field_pair.second);
} else if(field_pair.second.type == field_types::INT32) {
+ auto num_tree = numerical_index.at(field_name);
uint32_t value = document[field_name];
- index_int32_field(value, points, t, seq_id);
+ num_tree->insert(value, seq_id);
} else if(field_pair.second.type == field_types::INT64) {
+ auto num_tree = numerical_index.at(field_name);
uint64_t value = document[field_name];
- index_int64_field(value, points, t, seq_id);
+ num_tree->insert(value, seq_id);
} else if(field_pair.second.type == field_types::FLOAT) {
- float value = document[field_name];
- index_float_field(value, points, t, seq_id);
+ auto num_tree = numerical_index.at(field_name);
+ float fvalue = document[field_name];
+ int64_t value = float_to_in64_t(fvalue);
+ num_tree->insert(value, seq_id);
} else if(field_pair.second.type == field_types::BOOL) {
+ auto num_tree = numerical_index.at(field_name);
bool value = document[field_name];
- index_bool_field(value, points, t, seq_id);
+ num_tree->insert(value, seq_id);
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
+ art_tree *t = search_index.at(field_name);
std::vector<std::string> strings = document[field_name];
index_string_array_field(strings, points, t, seq_id, facet_id, field_pair.second);
} else if(field_pair.second.type == field_types::INT32_ARRAY) {
- std::vector<int32_t> values = document[field_name];
- index_int32_array_field(values, points, t, seq_id);
+ auto num_tree = numerical_index.at(field_name);
+ const std::vector<int32_t>& arr_values = document[field_name];
+ for(const int32_t value: arr_values) {
+ num_tree->insert(value, seq_id);
+ }
} else if(field_pair.second.type == field_types::INT64_ARRAY) {
- std::vector<int64_t> values = document[field_name];
- index_int64_array_field(values, points, t, seq_id);
+ auto& num_tree = numerical_index.at(field_name);
+ const std::vector<int64_t>& arr_values = document[field_name];
+ for(const int64_t value: arr_values) {
+ num_tree->insert(value, seq_id);
+ }
} else if(field_pair.second.type == field_types::FLOAT_ARRAY) {
- std::vector<float> values = document[field_name];
- index_float_array_field(values, points, t, seq_id);
+ const std::vector<float>& arr_values = document[field_name];
+ for(const float fvalue: arr_values) {
+ auto& num_tree = numerical_index.at(field_name);
+ int64_t value = float_to_in64_t(fvalue);
+ num_tree->insert(value, seq_id);
+ }
} else if(field_pair.second.type == field_types::BOOL_ARRAY) {
- std::vector<bool> values = document[field_name];
- index_bool_array_field(values, points, t, seq_id);
+ auto& num_tree = numerical_index.at(field_name);
+ const std::vector<bool>& arr_values = document[field_name];
+ for(const int64_t value: arr_values) {
+ num_tree->insert(value, seq_id);
+ }
}
// add numerical values automatically into sort index
@@ -334,35 +362,63 @@ void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_do
}
const auto& search_field = search_field_it->second;
-
- // Go through all the field names and find the keys+values so that they can be removed from in-memory index
- std::vector<std::string> reindex_tokens;
- std::vector<std::string> old_tokens;
- tokenize_doc_field(update_doc, search_field, reindex_tokens);
- tokenize_doc_field(old_doc, search_field, old_tokens);
-
- if(old_tokens.size() != reindex_tokens.size()) {
+ bool arrays_match = false;
+
+ if(search_field.is_string()) {
+ // Go through all the field names and find the keys+values so that they can be removed from in-memory index
+ std::vector<std::string> reindex_vals;
+ std::vector<std::string> old_vals;
+
+ tokenize_doc_field(update_doc, search_field, reindex_vals);
+ tokenize_doc_field(old_doc, search_field, old_vals);
+
+ arrays_match = _arrays_match<std::string>(reindex_vals, old_vals);
+
+ } else if(search_field.is_int32()) {
+ std::vector<int32_t> reindex_vals = search_field.is_single_integer() ?
+ std::vector<int32_t>{update_doc[field_name].get<int32_t>()} :
+ update_doc[field_name].get<std::vector<int32_t>>();
+ std::vector<int32_t> old_vals = search_field.is_single_integer() ?
+ std::vector<int32_t>{old_doc[field_name].get<int32_t>()} :
+ old_doc[field_name].get<std::vector<int32_t>>();
+
+ arrays_match = _arrays_match<int32_t>(reindex_vals, old_vals);
+ } else if(search_field.is_int64()) {
+ std::vector<int64_t> reindex_vals = search_field.is_single_integer() ?
+ std::vector<int64_t>{update_doc[field_name].get<int64_t>()} :
+ update_doc[field_name].get<std::vector<int64_t>>();
+ std::vector<int64_t> old_vals = search_field.is_single_integer() ?
+ std::vector<int64_t>{old_doc[field_name].get<int64_t>()} :
+ old_doc[field_name].get<std::vector<int64_t>>();
+
+ arrays_match = _arrays_match<int64_t>(reindex_vals, old_vals);
+ } else if(search_field.is_float()) {
+ std::vector<float> reindex_vals = search_field.is_single_float() ?
+ std::vector<float>{update_doc[field_name].get<float>()} :
+ update_doc[field_name].get<std::vector<float>>();
+ std::vector<float> old_vals = search_field.is_single_float() ?
+ std::vector<float>{old_doc[field_name].get<float>()} :
+ old_doc[field_name].get<std::vector<float>>();
+
+ arrays_match = _arrays_match<float>(reindex_vals, old_vals);
+ } else if(search_field.is_bool()) {
+ std::vector<bool> reindex_vals = search_field.is_single_bool() ?
+ std::vector<bool>{update_doc[field_name].get<bool>()} :
+ update_doc[field_name].get<std::vector<bool>>();
+ std::vector<bool> old_vals = search_field.is_single_bool() ?
+ std::vector<bool>{old_doc[field_name].get<bool>()} :
+ old_doc[field_name].get<std::vector<bool>>();
+
+ arrays_match = _arrays_match<bool>(reindex_vals, old_vals);
+ }
+
+ if(!arrays_match) {
++it;
- continue;
- }
-
- bool exact_match = true;
-
- for(size_t i=0; i<reindex_tokens.size(); i++) {
- const std::string& reindex_val = reindex_tokens[i];
- const std::string& old_val = old_tokens[i];
- if(reindex_val != old_val) {
- exact_match = false;
- break;
- }
- }
-
- if(exact_match) {
+ } else {
it = del_doc.erase(it);
update_doc.erase(field_name);
- } else {
- ++it;
}
+
}
}
@@ -446,98 +502,6 @@ void Index::insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
}
}
-void Index::index_int32_field(const int32_t value, int64_t score, art_tree *t, uint32_t seq_id) const {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
-
- encode_int32(value, key);
-
- uint32_t num_hits = 0;
- art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);
- if(leaf != NULL) {
- num_hits = leaf->values->ids.getLength();
- }
-
- num_hits += 1;
-
- art_document art_doc;
- art_doc.id = seq_id;
- art_doc.score = score;
- art_doc.offsets_len = 0;
- art_doc.offsets = nullptr;
-
- art_insert(t, key, KEY_LEN, &art_doc, num_hits);
-}
-
-void Index::index_int64_field(const int64_t value, int64_t score, art_tree *t, uint32_t seq_id) const {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
-
- encode_int64(value, key);
-
- uint32_t num_hits = 0;
- art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);
- if(leaf != NULL) {
- num_hits = leaf->values->ids.getLength();
- }
-
- num_hits += 1;
-
- art_document art_doc;
- art_doc.id = seq_id;
- art_doc.score = score;
- art_doc.offsets_len = 0;
- art_doc.offsets = nullptr;
-
- art_insert(t, key, KEY_LEN, &art_doc, num_hits);
-}
-
-void Index::index_bool_field(const bool value, const int64_t score, art_tree *t, uint32_t seq_id) const {
- const int KEY_LEN = 1;
- unsigned char key[KEY_LEN];
- key[0] = value ? '1' : '0';
-
- uint32_t num_hits = 0;
- art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);
- if(leaf != NULL) {
- num_hits = leaf->values->ids.getLength();
- }
-
- num_hits += 1;
-
- art_document art_doc;
- art_doc.id = seq_id;
- art_doc.score = score;
- art_doc.offsets_len = 0;
- art_doc.offsets = nullptr;
-
- art_insert(t, key, KEY_LEN, &art_doc, num_hits);
-}
-
-void Index::index_float_field(const float value, int64_t score, art_tree *t, uint32_t seq_id) const {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
-
- encode_float(value, key);
-
- uint32_t num_hits = 0;
- art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);
- if(leaf != NULL) {
- num_hits = leaf->values->ids.getLength();
- }
-
- num_hits += 1;
-
- art_document art_doc;
- art_doc.id = seq_id;
- art_doc.score = score;
- art_doc.offsets_len = 0;
- art_doc.offsets = nullptr;
-
- art_insert(t, key, KEY_LEN, &art_doc, num_hits);
-}
-
-
uint64_t Index::facet_token_hash(const field & a_field, const std::string &token) {
// for integer/float use their native values
uint64_t hash = 0;
@@ -637,34 +601,6 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c
insert_doc(score, t, seq_id, token_positions);
}
-void Index::index_int32_array_field(const std::vector<int32_t> & values, const int64_t score, art_tree *t,
- uint32_t seq_id) const {
- for(const int32_t value: values) {
- index_int32_field(value, score, t, seq_id);
- }
-}
-
-void Index::index_int64_array_field(const std::vector<int64_t> & values, const int64_t score, art_tree *t,
- uint32_t seq_id) const {
- for(const int64_t value: values) {
- index_int64_field(value, score, t, seq_id);
- }
-}
-
-void Index::index_bool_array_field(const std::vector<bool> & values, const int64_t score, art_tree *t,
- uint32_t seq_id) const {
- for(const bool value: values) {
- index_bool_field(value, score, t, seq_id);
- }
-}
-
-void Index::index_float_array_field(const std::vector<float> & values, const int64_t score, art_tree *t,
- uint32_t seq_id) const {
- for(const float value: values) {
- index_float_field(value, score, t, seq_id);
- }
-}
-
void Index::compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type) {
if(field_type == field_types::INT32 || field_type == field_types::INT32_ARRAY) {
int32_t val = raw_value;
@@ -959,11 +895,12 @@ void Index::search_candidates(const uint8_t & field_id,
result_ids = excluded_result_ids;
}
- /*LOG(INFO) << "n: " << n;
+ //LOG(INFO) << "n: " << n;
+ std::stringstream log_query;
for(size_t i=0; i < query_suggestion.size(); i++) {
- LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key << ", ids: "
- << query_suggestion[i]->values->ids.getLength();
- }*/
+ log_query << query_suggestion[i]->key << " ";
+ }
+
if(filter_ids != nullptr) {
// intersect once again with filter ids
@@ -992,6 +929,11 @@ void Index::search_candidates(const uint8_t & field_id,
score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,
groups_processed, result_ids, result_size);
+
+ /*if(result_size != 0) {
+ LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size;
+ }*/
+
delete[] result_ids;
}
@@ -1013,61 +955,59 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
for(size_t i = 0; i < filters.size(); i++) {
const filter & a_filter = filters[i];
- if(search_index.count(a_filter.field_name) == 0) {
+ bool has_search_index = search_index.count(a_filter.field_name) != 0 ||
+ numerical_index.count(a_filter.field_name) != 0;
+
+ if(!has_search_index) {
continue;
}
- art_tree* t = search_index.at(a_filter.field_name);
field f = search_schema.at(a_filter.field_name);
uint32_t* result_ids = nullptr;
size_t result_ids_len = 0;
if(f.is_integer()) {
- std::vector<const art_leaf*> leaves;
- std::vector<uint32_t> ids;
+ auto num_tree = numerical_index.at(a_filter.field_name);
size_t value_index = 0;
for(const std::string & filter_value: a_filter.values) {
if(f.type == field_types::INT32 || f.type == field_types::INT32_ARRAY) {
// check for comparator again
int32_t value = (int32_t) std::stoi(filter_value);
- art_int32_search(t, value, a_filter.comparators[value_index], leaves);
+ num_tree->search(a_filter.comparators[value_index], value, &result_ids, result_ids_len);
} else { // int64
int64_t value = (int64_t) std::stol(filter_value);
- art_int64_search(t, value, a_filter.comparators[value_index], leaves);
+ num_tree->search(a_filter.comparators[value_index], value, &result_ids, result_ids_len);
}
value_index++;
}
- result_ids = collate_leaf_ids(leaves, result_ids_len);
-
} else if(f.is_float()) {
- std::vector<const art_leaf*> leaves;
- std::vector<uint32_t> ids;
+ auto num_tree = numerical_index.at(a_filter.field_name);
size_t value_index = 0;
for(const std::string & filter_value: a_filter.values) {
float value = (float) std::atof(filter_value.c_str());
- art_float_search(t, value, a_filter.comparators[value_index], leaves);
+ int64_t float_int64 = float_to_in64_t(value);
+ num_tree->search(a_filter.comparators[value_index], float_int64, &result_ids, result_ids_len);
value_index++;
}
- result_ids = collate_leaf_ids(leaves, result_ids_len);
-
} else if(f.is_bool()) {
- std::vector<const art_leaf*> leaves;
+ auto num_tree = numerical_index.at(a_filter.field_name);
+ size_t value_index = 0;
for(const std::string & filter_value: a_filter.values) {
- art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) filter_value.c_str(),
- filter_value.length());
- if(leaf) {
- leaves.push_back(leaf);
- }
+ int64_t bool_int64 = (filter_value == "1") ? 1 : 0;
+ num_tree->search(a_filter.comparators[value_index], bool_int64, &result_ids, result_ids_len);
+ value_index++;
}
- result_ids = collate_leaf_ids(leaves, result_ids_len);
+
} else if(f.is_string()) {
+ art_tree* t = search_index.at(a_filter.field_name);
+
uint32_t* ids = nullptr;
size_t ids_size = 0;
@@ -1249,23 +1189,6 @@ void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,
}
}
-uint32_t* Index::collate_leaf_ids(const std::vector<const art_leaf *> &leaves, size_t& result_ids_len) const {
- std::vector<uint32_t> ids;
-
- for(const art_leaf* leaf: leaves) {
- uint32_t num_ids = leaf->values->ids.getLength();
- uint32_t* leaf_ids = leaf->values->ids.uncompress();
- std::copy(leaf_ids, leaf_ids + num_ids, std::back_inserter(ids));
- delete [] leaf_ids;
- }
-
- uint32_t* result_ids = new uint32_t[ids.size()];
- std::sort(ids.begin(), ids.end());
- std::copy(ids.begin(), ids.end(), result_ids);
- result_ids_len = ids.size();
- return result_ids;
-}
-
void Index::run_search() {
while(true) {
// wait until main thread sends data
@@ -1484,6 +1407,8 @@ void Index::search(Option<uint32_t> & outcome,
std::vector<std::string> search_tokens = q_include_tokens;
size_t num_tokens_dropped = 0;
+ //LOG(INFO) << "searching field! " << field;
+
search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,
field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
num_typos, searched_queries, topster, groups_processed, &all_result_ids, all_result_ids_len,
@@ -1491,6 +1416,8 @@ void Index::search(Option<uint32_t> & outcome,
collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);
}
}
+
+ delete [] exclude_token_ids;
}
do_facets(facets, facet_query, all_result_ids, all_result_ids_len);
@@ -1983,52 +1910,81 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
const auto& search_field = search_field_it->second;
// Go through all the field names and find the keys+values so that they can be removed from in-memory index
- std::vector<std::string> tokens;
- tokenize_doc_field(document, search_field, tokens);
+ if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {
+ std::vector<std::string> tokens;
+ tokenize_doc_field(document, search_field, tokens);
- for(auto & token: tokens) {
- const unsigned char *key;
- int key_len;
+ for(auto & token: tokens) {
+ const unsigned char *key = (const unsigned char *) token.c_str();
+ int key_len = (int) (token.length() + 1);
- if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {
- key = (const unsigned char *) token.c_str();
- key_len = (int) (token.length() + 1);
- } else {
- key = (const unsigned char *) token.c_str();
- key_len = (int) (token.length());
- }
-
- art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name), key, key_len);
- if(leaf != nullptr) {
- uint32_t doc_index = leaf->values->ids.indexOf(seq_id);
+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name), key, key_len);
+ if(leaf != nullptr) {
+ uint32_t doc_index = leaf->values->ids.indexOf(seq_id);
- if(doc_index == leaf->values->ids.getLength()) {
- // not found - happens when 2 tokens repeat in a field, e.g "is it or is is not?"
- continue;
- }
+ if (doc_index == leaf->values->ids.getLength()) {
+ // not found - happens when 2 tokens repeat in a field, e.g "is it or is is not?"
+ continue;
+ }
- uint32_t start_offset = leaf->values->offset_index.at(doc_index);
- uint32_t end_offset = (doc_index == leaf->values->ids.getLength() - 1) ?
- leaf->values->offsets.getLength() :
- leaf->values->offset_index.at(doc_index+1);
+ uint32_t start_offset = leaf->values->offset_index.at(doc_index);
+ uint32_t end_offset = (doc_index == leaf->values->ids.getLength() - 1) ?
+ leaf->values->offsets.getLength() :
+ leaf->values->offset_index.at(doc_index + 1);
- uint32_t doc_indices[1] = {doc_index};
- remove_and_shift_offset_index(leaf->values->offset_index, doc_indices, 1);
+ uint32_t doc_indices[1] = {doc_index};
+ remove_and_shift_offset_index(leaf->values->offset_index, doc_indices, 1);
- leaf->values->offsets.remove_index(start_offset, end_offset);
- leaf->values->ids.remove_value(seq_id);
+ leaf->values->offsets.remove_index(start_offset, end_offset);
+ leaf->values->ids.remove_value(seq_id);
- /*len = leaf->values->offset_index.getLength();
- for(auto i=0; i<len; i++) {
- LOG(INFO) << "i: " << i << ", val: " << leaf->values->offset_index.at(i);
- }
- LOG(INFO) << "----";*/
+ /*len = leaf->values->offset_index.getLength();
+ for(auto i=0; i<len; i++) {
+ LOG(INFO) << "i: " << i << ", val: " << leaf->values->offset_index.at(i);
+ }
+ LOG(INFO) << "----";*/
- if(leaf->values->ids.getLength() == 0) {
- art_values* values = (art_values*) art_delete(search_index.at(field_name), key, key_len);
- delete values;
+ if (leaf->values->ids.getLength() == 0) {
+ art_values *values = (art_values *) art_delete(search_index.at(field_name), key, key_len);
+ delete values;
+ }
}
}
+ } else if(search_field.is_int32()) {
+ const std::vector<int32_t>& values = search_field.is_single_integer() ?
+ std::vector<int32_t>{document[field_name].get<int32_t>()} :
+ document[field_name].get<std::vector<int32_t>>();
+ for(int32_t value: values) {
+ num_tree_t* num_tree = numerical_index.at(field_name);
+ num_tree->remove(value, seq_id);
+ }
+ } else if(search_field.is_int64()) {
+ const std::vector<int64_t>& values = search_field.is_single_integer() ?
+ std::vector<int64_t>{document[field_name].get<int64_t>()} :
+ document[field_name].get<std::vector<int64_t>>();
+ for(int64_t value: values) {
+ num_tree_t* num_tree = numerical_index.at(field_name);
+ num_tree->remove(value, seq_id);
+ }
+ } else if(search_field.is_float()) {
+ const std::vector<float>& values = search_field.is_single_float() ?
+ std::vector<float>{document[field_name].get<float>()} :
+ document[field_name].get<std::vector<float>>();
+ for(float value: values) {
+ num_tree_t* num_tree = numerical_index.at(field_name);
+ int64_t fintval = float_to_in64_t(value);
+ num_tree->remove(fintval, seq_id);
+ }
+ } else if(search_field.is_bool()) {
+
+ const std::vector<bool>& values = search_field.is_single_bool() ?
+ std::vector<bool>{document[field_name].get<bool>()} :
+ document[field_name].get<std::vector<bool>>();
+ for(bool value: values) {
+ num_tree_t* num_tree = numerical_index.at(field_name);
+ int64_t bool_int64 = value ? 1 : 0;
+ num_tree->remove(bool_int64, seq_id);
+ }
}
// remove facets
@@ -2059,62 +2015,6 @@ void Index::tokenize_doc_field(const nlohmann::json& document, const field& sear
for(const std::string & value: values) {
Tokenizer(value, true, true, !search_field.is_string()).tokenize(tokens);
}
- } else if(search_field.type == field_types::INT32) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- const int32_t& value = document[field_name].get<int32_t>();
- encode_int32(value, key);
- tokens.emplace_back((char*)key, KEY_LEN);
- } else if(search_field.type == field_types::INT32_ARRAY) {
- const std::vector<int32_t>& values = document[field_name].get<std::vector<int32_t>>();
- for(const int32_t value: values) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- encode_int32(value, key);
- tokens.emplace_back((char*)key, KEY_LEN);
- }
- } else if(search_field.type == field_types::INT64) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- const int64_t& value = document[field_name].get<int64_t>();
- encode_int64(value, key);
- tokens.emplace_back((char*)key, KEY_LEN);
- } else if(search_field.type == field_types::INT64_ARRAY) {
- const std::vector<int64_t>& values = document[field_name].get<std::vector<int64_t>>();
- for(const int64_t value: values) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- encode_int64(value, key);
- tokens.emplace_back((char*)key, KEY_LEN);
- }
- } else if(search_field.type == field_types::FLOAT) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- const int64_t& value = document[field_name].get<int64_t>();
- encode_float(value, key);
- tokens.emplace_back((char*)key, KEY_LEN);
- } else if(search_field.type == field_types::FLOAT_ARRAY) {
- const std::vector<float>& values = document[field_name].get<std::vector<float>>();
- for(const float value: values) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- encode_float(value, key);
- tokens.emplace_back((char*)key, KEY_LEN);
- }
- } else if(search_field.type == field_types::BOOL) {
- const int KEY_LEN = 1;
- unsigned char key[KEY_LEN];
- const bool& value = document[field_name].get<bool>();
- key[0] = value ? '1' : '0';
- tokens.emplace_back((char*)key, KEY_LEN);
- } else if(search_field.type == field_types::BOOL_ARRAY) {
- const std::vector<bool>& values = document[field_name].get<std::vector<bool>>();
- for(const bool value: values) {
- const int KEY_LEN = 1;
- unsigned char key[KEY_LEN];
- key[0] = value ? '1' : '0';
- tokens.emplace_back((char*)key, KEY_LEN);
- }
}
}
@@ -2126,3 +2026,7 @@ art_leaf* Index::get_token_leaf(const std::string & field_name, const unsigned c
const spp::sparse_hash_map<std::string, art_tree *> &Index::_get_search_index() const {
return search_index;
}
+
+const spp::sparse_hash_map<std::string, num_tree_t*>& Index::_get_numerical_index() const {
+ return numerical_index;
+}
diff --git a/test/art_test.cpp b/test/art_test.cpp
index b843978a..e3ab87c9 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -1226,6 +1226,27 @@ TEST(ArtTest, test_search_negative_int64) {
results.clear();
}
+TEST(ArtTest, test_search_negative_int64_large) {
+ art_tree t;
+ art_tree_init(&t);
+
+ art_document doc = get_document(1);
+ const int CHAR_LEN = 8;
+ unsigned char chars[CHAR_LEN];
+
+ const int64_t lmax = -1 * std::numeric_limits<std::int32_t>::max();
+
+ encode_int64(-2, chars);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) chars, CHAR_LEN, &doc, 1));
+
+ std::vector<const art_leaf *> results;
+
+ int res = art_int64_search(&t, 1577836800, GREATER_THAN, results);
+ //ASSERT_TRUE(res == 0);
+ //ASSERT_EQ(0, results.size());
+ //results.clear();
+}
+
TEST(ArtTest, test_int32_array) {
art_tree t;
art_tree_init(&t);
diff --git a/test/auth_manager_test.cpp b/test/auth_manager_test.cpp
index de32fce8..8ecf45c3 100644
--- a/test/auth_manager_test.cpp
+++ b/test/auth_manager_test.cpp
@@ -190,7 +190,16 @@ TEST_F(AuthManagerTest, VerifyAuthentication) {
}
TEST_F(AuthManagerTest, GenerationOfAPIAction) {
+ std::map<int64_t, int64_t> int64map;
+ int64map[1] = 1;
+ int64map[2] = 2;
+ int64map[3] = 3;
+
+ std::cout << (int64map.lower_bound(5) == int64map.end());
+ return ;
+
route_path rpath_search = route_path("GET", {"collections", ":collection", "documents", "search"}, nullptr, false, false);
+ route_path rpath_coll_create = route_path("POST", {"collections"}, nullptr, false, false);
route_path rpath_coll_get = route_path("GET", {"collections", ":collection"}, nullptr, false, false);
route_path rpath_coll_list = route_path("GET", {"collections"}, nullptr, false, false);
route_path rpath_keys_post = route_path("POST", {"keys"}, nullptr, false, false);
@@ -198,6 +207,7 @@ TEST_F(AuthManagerTest, GenerationOfAPIAction) {
route_path rpath_override_upsert = route_path("PUT", {"collections", ":collection", "overrides", ":id"}, nullptr, false, false);
ASSERT_STREQ("documents:search", rpath_search._get_action().c_str());
+ ASSERT_STREQ("collections:create", rpath_coll_create._get_action().c_str());
ASSERT_STREQ("collections:get", rpath_coll_get._get_action().c_str());
ASSERT_STREQ("collections:list", rpath_coll_list._get_action().c_str());
ASSERT_STREQ("keys:create", rpath_keys_post._get_action().c_str());
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index 8e101cd3..b488ea16 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -306,5 +306,38 @@ TEST_F(CollectionSortingTest, ThreeSortFieldsLimit) {
ASSERT_FALSE(res_op.ok());
ASSERT_STREQ("Only upto 3 sort_by fields can be specified.", res_op.error().c_str());
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionSortingTest, NegativeInt64Value) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT64, false),
+ };
+
+ coll1 = collectionManager.get_collection("coll1");
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
+ }
+
+ nlohmann::json doc1;
+
+ doc1["id"] = "100";
+ doc1["title"] = "The quick brown fox";
+ doc1["points"] = -2678400;
+
+ coll1->add(doc1.dump());
+
+ std::vector<sort_by> sort_fields_desc = {
+ sort_by("points", "DESC")
+ };
+
+ query_fields = {"title"};
+ auto res = coll1->search("*", query_fields, "points:>=1577836800", {}, sort_fields_desc, 0, 10, 1, FREQUENCY,
+ false).get();
+
+ ASSERT_EQ(0, res["found"].get<size_t>());
+
collectionManager.drop_collection("coll1");
}
\ No newline at end of file
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 503bf0aa..63b2701b 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1772,18 +1772,20 @@ TEST_F(CollectionTest, DeletionOfDocumentArrayFields) {
// also assert against the actual index
Index *index = coll1->_get_indexes()[0]; // seq id will always be zero for first document
auto search_index = index->_get_search_index();
+ auto numerical_index = index->_get_numerical_index();
auto strarray_tree = search_index["strarray"];
- auto int32array_tree = search_index["int32array"];
- auto int64array_tree = search_index["int64array"];
- auto floatarray_tree = search_index["floatarray"];
- auto boolarray_tree = search_index["boolarray"];
+ auto int32array_tree = numerical_index["int32array"];
+ auto int64array_tree = numerical_index["int64array"];
+ auto floatarray_tree = numerical_index["floatarray"];
+ auto boolarray_tree = numerical_index["boolarray"];
ASSERT_EQ(0, art_size(strarray_tree));
- ASSERT_EQ(0, art_size(int32array_tree));
- ASSERT_EQ(0, art_size(int64array_tree));
- ASSERT_EQ(0, art_size(floatarray_tree));
- ASSERT_EQ(0, art_size(boolarray_tree));
+
+ ASSERT_EQ(0, int32array_tree->size());
+ ASSERT_EQ(0, int64array_tree->size());
+ ASSERT_EQ(0, floatarray_tree->size());
+ ASSERT_EQ(0, boolarray_tree->size());
collectionManager.drop_collection("coll1");
}
@@ -2005,6 +2007,8 @@ TEST_F(CollectionTest, SearchHighlightShouldUseHighlightTags) {
ASSERT_STREQ("The quick brown fox jumped over the <em class=\"h\">lazy</em> fox. ",
res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
}
TEST_F(CollectionTest, SearchHighlightWithNewLine) {