Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Speed up numerical filter + fixed edge case with -ve value.
kishorenc6883b4db36d9
diff --git a/include/array_base.h b/include/array_base.hindex d4eae6a8..c53e6b12 100644--- a/include/array_base.h+++ b/include/array_base.h@@ -25,7 +25,7 @@ protected:}public:- array_base(const uint32_t n=2) {+ explicit array_base(const uint32_t n=2) {size_bytes = METADATA_OVERHEAD + (n * FOR_ELE_SIZE);in = (uint8_t *) malloc(size_bytes * sizeof *in);memset(in, 0, size_bytes);diff --git a/include/index.h b/include/index.hindex 33913276..a0d12e7f 100644--- a/include/index.h+++ b/include/index.h@@ -15,6 +15,7 @@#include <option.h>#include <set>#include "string_utils.h"+#include "num_tree.h"struct token_candidates {std::string token;@@ -135,6 +136,8 @@ private:spp::sparse_hash_map<std::string, art_tree*> search_index;+ spp::sparse_hash_map<std::string, num_tree_t*> numerical_index;+// seq_id => (facet => values)spp::sparse_hash_map<uint32_t, std::vector<std::vector<uint64_t>>> facet_index_v2;@@ -189,27 +192,9 @@ private:void index_string_array_field(const std::vector<std::string> & strings, const int64_t score, art_tree *t,uint32_t seq_id, int facet_id, const field & a_field);- void index_int32_field(const int32_t value, const int64_t score, art_tree *t, uint32_t seq_id) const;-- void index_int64_field(const int64_t value, const int64_t score, art_tree *t, uint32_t seq_id) const;-- void index_float_field(const float value, const int64_t score, art_tree *t, uint32_t seq_id) const;-- void index_bool_field(const bool value, const int64_t score, art_tree *t, uint32_t seq_id) const;-- void index_int32_array_field(const std::vector<int32_t> & values, const int64_t score, art_tree *t, uint32_t seq_id) const;-- void index_int64_array_field(const std::vector<int64_t> & values, const int64_t score, art_tree *t, uint32_t seq_id) const;-- void index_float_array_field(const std::vector<float> & values, const int64_t score, art_tree *t, uint32_t seq_id) const;-- void index_bool_array_field(const std::vector<bool> & values, const int64_t score, art_tree *t, uint32_t seq_id) const;-void remove_and_shift_offset_index(sorted_array& offset_index, const uint32_t* indices_sorted,const uint32_t indices_length);- uint32_t* collate_leaf_ids(const std::vector<const art_leaf *> &leaves, size_t& result_ids_len) const;-void collate_included_ids(const std::vector<std::string>& q_included_tokens,const std::string & field, const uint8_t field_id,const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,@@ -289,6 +274,8 @@ public:const spp::sparse_hash_map<std::string, art_tree *> &_get_search_index() const;+ const spp::sparse_hash_map<std::string, num_tree_t*>& _get_numerical_index() const;+// for limiting number of results on multiple candidates / query rewritesenum {TYPO_TOKENS_THRESHOLD = 100};@@ -328,5 +315,22 @@ public:void scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, nlohmann::json& old_doc);void tokenize_doc_field(const nlohmann::json& document, const field& search_field, std::vector<std::string>& tokens);++ template<typename T>+ bool _arrays_match(std::vector<T> reindex_vals, std::vector<T> old_vals) {+ if(old_vals.size() != reindex_vals.size()) {+ return false;+ }++ for(size_t i=0; i < reindex_vals.size(); i++) {+ const T& reindex_val = reindex_vals[i];+ const T& old_val = old_vals[i];+ if(reindex_val != old_val) {+ return false;+ }+ }++ return true;+ }};diff --git a/include/num_tree.h b/include/num_tree.hnew file mode 100644index 00000000..6f99f822--- /dev/null+++ b/include/num_tree.h@@ -0,0 +1,121 @@+#pragma once++#include <map>+#include "sparsepp.h"+#include "sorted_array.h"+#include "array_utils.h"++class num_tree_t {+private:+ std::map<int64_t, sorted_array*> int64map;++public:++ ~num_tree_t() {+ for(auto& kv: int64map) {+ delete kv.second;+ }+ }++ void insert(int64_t value, uint32_t id) {+ if(int64map.count(value) == 0) {+ int64map.emplace(value, new sorted_array);+ }++ int64map[value]->append(id);+ }++ void search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids, size_t& ids_len) {+ if(int64map.empty()) {+ return ;+ }++ if(comparator == EQUALS) {+ const auto& it = int64map.find(value);+ if(it != int64map.end()) {+ uint32_t *out = nullptr;+ uint32_t* val_ids = it->second->uncompress();+ ids_len = ArrayUtils::or_scalar(val_ids, it->second->getLength(),+ *ids, ids_len, &out);+ delete[] *ids;+ *ids = out;+ delete[] val_ids;+ }+ } else if(comparator == GREATER_THAN || comparator == GREATER_THAN_EQUALS) {+ auto it = int64map.lower_bound(value); // iter values will be >= value++ if(it == int64map.end()) {+ return ;+ }++ if(comparator == GREATER_THAN && it->first == value) {+ it++;+ }++ while(it != int64map.end()) {+ uint32_t *out = nullptr;+ uint32_t *val_ids = it->second->uncompress();+ ids_len = ArrayUtils::or_scalar(val_ids, it->second->getLength(),+ *ids, ids_len, &out);+ delete [] val_ids;+ delete [] *ids;+ *ids = out;+ it++;+ }+ } else if(comparator == LESS_THAN || comparator == LESS_THAN_EQUALS) {+ auto max_iter = int64map.lower_bound(value); // iter values will be >= value++ if(comparator == LESS_THAN) {+ if(max_iter == int64map.end()) {+ max_iter--;+ }++ else if(max_iter == int64map.begin() && max_iter->first != value) {+ return ;+ }++ else if(max_iter != int64map.begin() && max_iter->first >= value) {+ max_iter--;+ }+ } else {+ if(max_iter == int64map.end()) {+ max_iter--;+ }+ }++ auto iter = int64map.begin();++ while(true) {+ uint32_t* out = nullptr;+ uint32_t* val_ids = iter->second->uncompress();+ ids_len = ArrayUtils::or_scalar(val_ids, iter->second->getLength(),+ *ids, ids_len, &out);+ delete[] val_ids;+ delete[] *ids;+ *ids = out;++ if(iter == max_iter) {+ break;+ }++ iter++;+ }+ }+ }++ void remove(uint64_t value, uint32_t id) {+ if(int64map.count(value) != 0) {+ sorted_array* arr = int64map[value];+ arr->remove_value(id);+ if(arr->getLength() == 0) {+ delete arr;+ int64map.erase(value);+ }+ }+ }++ size_t size() {+ return int64map.size();+ }++};\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex cfe5d908..2d7c9873 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1796,6 +1796,7 @@ Option<bool> Collection::parse_filter_query(const std::string& simple_filter_queif(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {std::vector<std::string> filter_values;StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");+ f = {field_name, {}, {}};for(std::string & filter_value: filter_values) {if(filter_value != "true" && filter_value != "false") {@@ -1804,9 +1805,9 @@ Option<bool> Collection::parse_filter_query(const std::string& simple_filter_que}filter_value = (filter_value == "true") ? "1" : "0";+ f.values.push_back(filter_value);+ f.comparators.push_back(EQUALS);}-- f = {field_name, filter_values, {EQUALS}};} else {if(raw_value != "true" && raw_value != "false") {return Option<bool>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");diff --git a/src/index.cpp b/src/index.cppindex f1726a23..7db552ad 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -15,16 +15,21 @@ Index::Index(const std::string name, const std::unordered_map<std::string, fieldstd::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema):name(name), search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema) {- for(const auto & pair: search_schema) {- art_tree *t = new art_tree;- art_tree_init(t);- search_index.emplace(pair.first, t);+ for(const auto & fname_field: search_schema) {+ if(fname_field.second.is_string()) {+ art_tree *t = new art_tree;+ art_tree_init(t);+ search_index.emplace(fname_field.first, t);+ } else {+ num_tree_t* num_tree = new num_tree_t;+ numerical_index.emplace(fname_field.first, num_tree);+ }// initialize for non-string facet fields- if(pair.second.facet && !pair.second.is_string()) {+ if(fname_field.second.facet && !fname_field.second.is_string()) {art_tree *ft = new art_tree;art_tree_init(ft);- search_index.emplace(pair.second.faceted_name(), ft);+ search_index.emplace(fname_field.second.faceted_name(), ft);}}@@ -47,6 +52,11 @@ Index::~Index() {name_tree.second = nullptr;}+ for(auto & name_tree: numerical_index) {+ delete name_tree.second;+ name_tree.second = nullptr;+ }+search_index.clear();for(auto & name_map: sort_index) {@@ -164,38 +174,56 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t}}- art_tree *t = search_index.at(field_name);-if(field_pair.second.type == field_types::STRING) {+ art_tree *t = search_index.at(field_name);const std::string & text = document[field_name];index_string_field(text, points, t, seq_id, facet_id, field_pair.second);} else if(field_pair.second.type == field_types::INT32) {+ auto num_tree = numerical_index.at(field_name);uint32_t value = document[field_name];- index_int32_field(value, points, t, seq_id);+ num_tree->insert(value, seq_id);} else if(field_pair.second.type == field_types::INT64) {+ auto num_tree = numerical_index.at(field_name);uint64_t value = document[field_name];- index_int64_field(value, points, t, seq_id);+ num_tree->insert(value, seq_id);} else if(field_pair.second.type == field_types::FLOAT) {- float value = document[field_name];- index_float_field(value, points, t, seq_id);+ auto num_tree = numerical_index.at(field_name);+ float fvalue = document[field_name];+ int64_t value = float_to_in64_t(fvalue);+ num_tree->insert(value, seq_id);} else if(field_pair.second.type == field_types::BOOL) {+ auto num_tree = numerical_index.at(field_name);bool value = document[field_name];- index_bool_field(value, points, t, seq_id);+ num_tree->insert(value, seq_id);} else if(field_pair.second.type == field_types::STRING_ARRAY) {+ art_tree *t = search_index.at(field_name);std::vector<std::string> strings = document[field_name];index_string_array_field(strings, points, t, seq_id, facet_id, field_pair.second);} else if(field_pair.second.type == field_types::INT32_ARRAY) {- std::vector<int32_t> values = document[field_name];- index_int32_array_field(values, points, t, seq_id);+ auto num_tree = numerical_index.at(field_name);+ const std::vector<int32_t>& arr_values = document[field_name];+ for(const int32_t value: arr_values) {+ num_tree->insert(value, seq_id);+ }} else if(field_pair.second.type == field_types::INT64_ARRAY) {- std::vector<int64_t> values = document[field_name];- index_int64_array_field(values, points, t, seq_id);+ auto& num_tree = numerical_index.at(field_name);+ const std::vector<int64_t>& arr_values = document[field_name];+ for(const int64_t value: arr_values) {+ num_tree->insert(value, seq_id);+ }} else if(field_pair.second.type == field_types::FLOAT_ARRAY) {- std::vector<float> values = document[field_name];- index_float_array_field(values, points, t, seq_id);+ const std::vector<float>& arr_values = document[field_name];+ for(const float fvalue: arr_values) {+ auto& num_tree = numerical_index.at(field_name);+ int64_t value = float_to_in64_t(fvalue);+ num_tree->insert(value, seq_id);+ }} else if(field_pair.second.type == field_types::BOOL_ARRAY) {- std::vector<bool> values = document[field_name];- index_bool_array_field(values, points, t, seq_id);+ auto& num_tree = numerical_index.at(field_name);+ const std::vector<bool>& arr_values = document[field_name];+ for(const int64_t value: arr_values) {+ num_tree->insert(value, seq_id);+ }}// add numerical values automatically into sort index@@ -334,35 +362,63 @@ void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_do}const auto& search_field = search_field_it->second;-- // Go through all the field names and find the keys+values so that they can be removed from in-memory index- std::vector<std::string> reindex_tokens;- std::vector<std::string> old_tokens;- tokenize_doc_field(update_doc, search_field, reindex_tokens);- tokenize_doc_field(old_doc, search_field, old_tokens);-- if(old_tokens.size() != reindex_tokens.size()) {+ bool arrays_match = false;++ if(search_field.is_string()) {+ // Go through all the field names and find the keys+values so that they can be removed from in-memory index+ std::vector<std::string> reindex_vals;+ std::vector<std::string> old_vals;++ tokenize_doc_field(update_doc, search_field, reindex_vals);+ tokenize_doc_field(old_doc, search_field, old_vals);++ arrays_match = _arrays_match<std::string>(reindex_vals, old_vals);++ } else if(search_field.is_int32()) {+ std::vector<int32_t> reindex_vals = search_field.is_single_integer() ?+ std::vector<int32_t>{update_doc[field_name].get<int32_t>()} :+ update_doc[field_name].get<std::vector<int32_t>>();+ std::vector<int32_t> old_vals = search_field.is_single_integer() ?+ std::vector<int32_t>{old_doc[field_name].get<int32_t>()} :+ old_doc[field_name].get<std::vector<int32_t>>();++ arrays_match = _arrays_match<int32_t>(reindex_vals, old_vals);+ } else if(search_field.is_int64()) {+ std::vector<int64_t> reindex_vals = search_field.is_single_integer() ?+ std::vector<int64_t>{update_doc[field_name].get<int64_t>()} :+ update_doc[field_name].get<std::vector<int64_t>>();+ std::vector<int64_t> old_vals = search_field.is_single_integer() ?+ std::vector<int64_t>{old_doc[field_name].get<int64_t>()} :+ old_doc[field_name].get<std::vector<int64_t>>();++ arrays_match = _arrays_match<int64_t>(reindex_vals, old_vals);+ } else if(search_field.is_float()) {+ std::vector<float> reindex_vals = search_field.is_single_float() ?+ std::vector<float>{update_doc[field_name].get<float>()} :+ update_doc[field_name].get<std::vector<float>>();+ std::vector<float> old_vals = search_field.is_single_float() ?+ std::vector<float>{old_doc[field_name].get<float>()} :+ old_doc[field_name].get<std::vector<float>>();++ arrays_match = _arrays_match<float>(reindex_vals, old_vals);+ } else if(search_field.is_bool()) {+ std::vector<bool> reindex_vals = search_field.is_single_bool() ?+ std::vector<bool>{update_doc[field_name].get<bool>()} :+ update_doc[field_name].get<std::vector<bool>>();+ std::vector<bool> old_vals = search_field.is_single_bool() ?+ std::vector<bool>{old_doc[field_name].get<bool>()} :+ old_doc[field_name].get<std::vector<bool>>();++ arrays_match = _arrays_match<bool>(reindex_vals, old_vals);+ }++ if(!arrays_match) {++it;- continue;- }-- bool exact_match = true;-- for(size_t i=0; i<reindex_tokens.size(); i++) {- const std::string& reindex_val = reindex_tokens[i];- const std::string& old_val = old_tokens[i];- if(reindex_val != old_val) {- exact_match = false;- break;- }- }-- if(exact_match) {+ } else {it = del_doc.erase(it);update_doc.erase(field_name);- } else {- ++it;}+}}@@ -446,98 +502,6 @@ void Index::insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,}}-void Index::index_int32_field(const int32_t value, int64_t score, art_tree *t, uint32_t seq_id) const {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];-- encode_int32(value, key);-- uint32_t num_hits = 0;- art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);- if(leaf != NULL) {- num_hits = leaf->values->ids.getLength();- }-- num_hits += 1;-- art_document art_doc;- art_doc.id = seq_id;- art_doc.score = score;- art_doc.offsets_len = 0;- art_doc.offsets = nullptr;-- art_insert(t, key, KEY_LEN, &art_doc, num_hits);-}--void Index::index_int64_field(const int64_t value, int64_t score, art_tree *t, uint32_t seq_id) const {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];-- encode_int64(value, key);-- uint32_t num_hits = 0;- art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);- if(leaf != NULL) {- num_hits = leaf->values->ids.getLength();- }-- num_hits += 1;-- art_document art_doc;- art_doc.id = seq_id;- art_doc.score = score;- art_doc.offsets_len = 0;- art_doc.offsets = nullptr;-- art_insert(t, key, KEY_LEN, &art_doc, num_hits);-}--void Index::index_bool_field(const bool value, const int64_t score, art_tree *t, uint32_t seq_id) const {- const int KEY_LEN = 1;- unsigned char key[KEY_LEN];- key[0] = value ? '1' : '0';-- uint32_t num_hits = 0;- art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);- if(leaf != NULL) {- num_hits = leaf->values->ids.getLength();- }-- num_hits += 1;-- art_document art_doc;- art_doc.id = seq_id;- art_doc.score = score;- art_doc.offsets_len = 0;- art_doc.offsets = nullptr;-- art_insert(t, key, KEY_LEN, &art_doc, num_hits);-}--void Index::index_float_field(const float value, int64_t score, art_tree *t, uint32_t seq_id) const {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];-- encode_float(value, key);-- uint32_t num_hits = 0;- art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);- if(leaf != NULL) {- num_hits = leaf->values->ids.getLength();- }-- num_hits += 1;-- art_document art_doc;- art_doc.id = seq_id;- art_doc.score = score;- art_doc.offsets_len = 0;- art_doc.offsets = nullptr;-- art_insert(t, key, KEY_LEN, &art_doc, num_hits);-}--uint64_t Index::facet_token_hash(const field & a_field, const std::string &token) {// for integer/float use their native valuesuint64_t hash = 0;@@ -637,34 +601,6 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, cinsert_doc(score, t, seq_id, token_positions);}-void Index::index_int32_array_field(const std::vector<int32_t> & values, const int64_t score, art_tree *t,- uint32_t seq_id) const {- for(const int32_t value: values) {- index_int32_field(value, score, t, seq_id);- }-}--void Index::index_int64_array_field(const std::vector<int64_t> & values, const int64_t score, art_tree *t,- uint32_t seq_id) const {- for(const int64_t value: values) {- index_int64_field(value, score, t, seq_id);- }-}--void Index::index_bool_array_field(const std::vector<bool> & values, const int64_t score, art_tree *t,- uint32_t seq_id) const {- for(const bool value: values) {- index_bool_field(value, score, t, seq_id);- }-}--void Index::index_float_array_field(const std::vector<float> & values, const int64_t score, art_tree *t,- uint32_t seq_id) const {- for(const float value: values) {- index_float_field(value, score, t, seq_id);- }-}-void Index::compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type) {if(field_type == field_types::INT32 || field_type == field_types::INT32_ARRAY) {int32_t val = raw_value;@@ -959,11 +895,12 @@ void Index::search_candidates(const uint8_t & field_id,result_ids = excluded_result_ids;}- /*LOG(INFO) << "n: " << n;+ //LOG(INFO) << "n: " << n;+ std::stringstream log_query;for(size_t i=0; i < query_suggestion.size(); i++) {- LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key << ", ids: "- << query_suggestion[i]->values->ids.getLength();- }*/+ log_query << query_suggestion[i]->key << " ";+ }+if(filter_ids != nullptr) {// intersect once again with filter ids@@ -992,6 +929,11 @@ void Index::search_candidates(const uint8_t & field_id,score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,groups_processed, result_ids, result_size);++ /*if(result_size != 0) {+ LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size;+ }*/+delete[] result_ids;}@@ -1013,61 +955,59 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vectofor(size_t i = 0; i < filters.size(); i++) {const filter & a_filter = filters[i];- if(search_index.count(a_filter.field_name) == 0) {+ bool has_search_index = search_index.count(a_filter.field_name) != 0 ||+ numerical_index.count(a_filter.field_name) != 0;++ if(!has_search_index) {continue;}- art_tree* t = search_index.at(a_filter.field_name);field f = search_schema.at(a_filter.field_name);uint32_t* result_ids = nullptr;size_t result_ids_len = 0;if(f.is_integer()) {- std::vector<const art_leaf*> leaves;- std::vector<uint32_t> ids;+ auto num_tree = numerical_index.at(a_filter.field_name);size_t value_index = 0;for(const std::string & filter_value: a_filter.values) {if(f.type == field_types::INT32 || f.type == field_types::INT32_ARRAY) {// check for comparator againint32_t value = (int32_t) std::stoi(filter_value);- art_int32_search(t, value, a_filter.comparators[value_index], leaves);+ num_tree->search(a_filter.comparators[value_index], value, &result_ids, result_ids_len);} else { // int64int64_t value = (int64_t) std::stol(filter_value);- art_int64_search(t, value, a_filter.comparators[value_index], leaves);+ num_tree->search(a_filter.comparators[value_index], value, &result_ids, result_ids_len);}value_index++;}- result_ids = collate_leaf_ids(leaves, result_ids_len);-} else if(f.is_float()) {- std::vector<const art_leaf*> leaves;- std::vector<uint32_t> ids;+ auto num_tree = numerical_index.at(a_filter.field_name);size_t value_index = 0;for(const std::string & filter_value: a_filter.values) {float value = (float) std::atof(filter_value.c_str());- art_float_search(t, value, a_filter.comparators[value_index], leaves);+ int64_t float_int64 = float_to_in64_t(value);+ num_tree->search(a_filter.comparators[value_index], float_int64, &result_ids, result_ids_len);value_index++;}- result_ids = collate_leaf_ids(leaves, result_ids_len);-} else if(f.is_bool()) {- std::vector<const art_leaf*> leaves;+ auto num_tree = numerical_index.at(a_filter.field_name);+ size_t value_index = 0;for(const std::string & filter_value: a_filter.values) {- art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) filter_value.c_str(),- filter_value.length());- if(leaf) {- leaves.push_back(leaf);- }+ int64_t bool_int64 = (filter_value == "1") ? 1 : 0;+ num_tree->search(a_filter.comparators[value_index], bool_int64, &result_ids, result_ids_len);+ value_index++;}- result_ids = collate_leaf_ids(leaves, result_ids_len);+} else if(f.is_string()) {+ art_tree* t = search_index.at(a_filter.field_name);+uint32_t* ids = nullptr;size_t ids_size = 0;@@ -1249,23 +1189,6 @@ void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,}}-uint32_t* Index::collate_leaf_ids(const std::vector<const art_leaf *> &leaves, size_t& result_ids_len) const {- std::vector<uint32_t> ids;-- for(const art_leaf* leaf: leaves) {- uint32_t num_ids = leaf->values->ids.getLength();- uint32_t* leaf_ids = leaf->values->ids.uncompress();- std::copy(leaf_ids, leaf_ids + num_ids, std::back_inserter(ids));- delete [] leaf_ids;- }-- uint32_t* result_ids = new uint32_t[ids.size()];- std::sort(ids.begin(), ids.end());- std::copy(ids.begin(), ids.end(), result_ids);- result_ids_len = ids.size();- return result_ids;-}-void Index::run_search() {while(true) {// wait until main thread sends data@@ -1484,6 +1407,8 @@ void Index::search(Option<uint32_t> & outcome,std::vector<std::string> search_tokens = q_include_tokens;size_t num_tokens_dropped = 0;+ //LOG(INFO) << "searching field! " << field;+search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,num_typos, searched_queries, topster, groups_processed, &all_result_ids, all_result_ids_len,@@ -1491,6 +1416,8 @@ void Index::search(Option<uint32_t> & outcome,collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);}}++ delete [] exclude_token_ids;}do_facets(facets, facet_query, all_result_ids, all_result_ids_len);@@ -1983,52 +1910,81 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & docconst auto& search_field = search_field_it->second;// Go through all the field names and find the keys+values so that they can be removed from in-memory index- std::vector<std::string> tokens;- tokenize_doc_field(document, search_field, tokens);+ if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {+ std::vector<std::string> tokens;+ tokenize_doc_field(document, search_field, tokens);- for(auto & token: tokens) {- const unsigned char *key;- int key_len;+ for(auto & token: tokens) {+ const unsigned char *key = (const unsigned char *) token.c_str();+ int key_len = (int) (token.length() + 1);- if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {- key = (const unsigned char *) token.c_str();- key_len = (int) (token.length() + 1);- } else {- key = (const unsigned char *) token.c_str();- key_len = (int) (token.length());- }-- art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name), key, key_len);- if(leaf != nullptr) {- uint32_t doc_index = leaf->values->ids.indexOf(seq_id);+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name), key, key_len);+ if(leaf != nullptr) {+ uint32_t doc_index = leaf->values->ids.indexOf(seq_id);- if(doc_index == leaf->values->ids.getLength()) {- // not found - happens when 2 tokens repeat in a field, e.g "is it or is is not?"- continue;- }+ if (doc_index == leaf->values->ids.getLength()) {+ // not found - happens when 2 tokens repeat in a field, e.g "is it or is is not?"+ continue;+ }- uint32_t start_offset = leaf->values->offset_index.at(doc_index);- uint32_t end_offset = (doc_index == leaf->values->ids.getLength() - 1) ?- leaf->values->offsets.getLength() :- leaf->values->offset_index.at(doc_index+1);+ uint32_t start_offset = leaf->values->offset_index.at(doc_index);+ uint32_t end_offset = (doc_index == leaf->values->ids.getLength() - 1) ?+ leaf->values->offsets.getLength() :+ leaf->values->offset_index.at(doc_index + 1);- uint32_t doc_indices[1] = {doc_index};- remove_and_shift_offset_index(leaf->values->offset_index, doc_indices, 1);+ uint32_t doc_indices[1] = {doc_index};+ remove_and_shift_offset_index(leaf->values->offset_index, doc_indices, 1);- leaf->values->offsets.remove_index(start_offset, end_offset);- leaf->values->ids.remove_value(seq_id);+ leaf->values->offsets.remove_index(start_offset, end_offset);+ leaf->values->ids.remove_value(seq_id);- /*len = leaf->values->offset_index.getLength();- for(auto i=0; i<len; i++) {- LOG(INFO) << "i: " << i << ", val: " << leaf->values->offset_index.at(i);- }- LOG(INFO) << "----";*/+ /*len = leaf->values->offset_index.getLength();+ for(auto i=0; i<len; i++) {+ LOG(INFO) << "i: " << i << ", val: " << leaf->values->offset_index.at(i);+ }+ LOG(INFO) << "----";*/- if(leaf->values->ids.getLength() == 0) {- art_values* values = (art_values*) art_delete(search_index.at(field_name), key, key_len);- delete values;+ if (leaf->values->ids.getLength() == 0) {+ art_values *values = (art_values *) art_delete(search_index.at(field_name), key, key_len);+ delete values;+ }}}+ } else if(search_field.is_int32()) {+ const std::vector<int32_t>& values = search_field.is_single_integer() ?+ std::vector<int32_t>{document[field_name].get<int32_t>()} :+ document[field_name].get<std::vector<int32_t>>();+ for(int32_t value: values) {+ num_tree_t* num_tree = numerical_index.at(field_name);+ num_tree->remove(value, seq_id);+ }+ } else if(search_field.is_int64()) {+ const std::vector<int64_t>& values = search_field.is_single_integer() ?+ std::vector<int64_t>{document[field_name].get<int64_t>()} :+ document[field_name].get<std::vector<int64_t>>();+ for(int64_t value: values) {+ num_tree_t* num_tree = numerical_index.at(field_name);+ num_tree->remove(value, seq_id);+ }+ } else if(search_field.is_float()) {+ const std::vector<float>& values = search_field.is_single_float() ?+ std::vector<float>{document[field_name].get<float>()} :+ document[field_name].get<std::vector<float>>();+ for(float value: values) {+ num_tree_t* num_tree = numerical_index.at(field_name);+ int64_t fintval = float_to_in64_t(value);+ num_tree->remove(fintval, seq_id);+ }+ } else if(search_field.is_bool()) {++ const std::vector<bool>& values = search_field.is_single_bool() ?+ std::vector<bool>{document[field_name].get<bool>()} :+ document[field_name].get<std::vector<bool>>();+ for(bool value: values) {+ num_tree_t* num_tree = numerical_index.at(field_name);+ int64_t bool_int64 = value ? 1 : 0;+ num_tree->remove(bool_int64, seq_id);+ }}// remove facets@@ -2059,62 +2015,6 @@ void Index::tokenize_doc_field(const nlohmann::json& document, const field& searfor(const std::string & value: values) {Tokenizer(value, true, true, !search_field.is_string()).tokenize(tokens);}- } else if(search_field.type == field_types::INT32) {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];- const int32_t& value = document[field_name].get<int32_t>();- encode_int32(value, key);- tokens.emplace_back((char*)key, KEY_LEN);- } else if(search_field.type == field_types::INT32_ARRAY) {- const std::vector<int32_t>& values = document[field_name].get<std::vector<int32_t>>();- for(const int32_t value: values) {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];- encode_int32(value, key);- tokens.emplace_back((char*)key, KEY_LEN);- }- } else if(search_field.type == field_types::INT64) {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];- const int64_t& value = document[field_name].get<int64_t>();- encode_int64(value, key);- tokens.emplace_back((char*)key, KEY_LEN);- } else if(search_field.type == field_types::INT64_ARRAY) {- const std::vector<int64_t>& values = document[field_name].get<std::vector<int64_t>>();- for(const int64_t value: values) {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];- encode_int64(value, key);- tokens.emplace_back((char*)key, KEY_LEN);- }- } else if(search_field.type == field_types::FLOAT) {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];- const int64_t& value = document[field_name].get<int64_t>();- encode_float(value, key);- tokens.emplace_back((char*)key, KEY_LEN);- } else if(search_field.type == field_types::FLOAT_ARRAY) {- const std::vector<float>& values = document[field_name].get<std::vector<float>>();- for(const float value: values) {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];- encode_float(value, key);- tokens.emplace_back((char*)key, KEY_LEN);- }- } else if(search_field.type == field_types::BOOL) {- const int KEY_LEN = 1;- unsigned char key[KEY_LEN];- const bool& value = document[field_name].get<bool>();- key[0] = value ? '1' : '0';- tokens.emplace_back((char*)key, KEY_LEN);- } else if(search_field.type == field_types::BOOL_ARRAY) {- const std::vector<bool>& values = document[field_name].get<std::vector<bool>>();- for(const bool value: values) {- const int KEY_LEN = 1;- unsigned char key[KEY_LEN];- key[0] = value ? '1' : '0';- tokens.emplace_back((char*)key, KEY_LEN);- }}}@@ -2126,3 +2026,7 @@ art_leaf* Index::get_token_leaf(const std::string & field_name, const unsigned cconst spp::sparse_hash_map<std::string, art_tree *> &Index::_get_search_index() const {return search_index;}++const spp::sparse_hash_map<std::string, num_tree_t*>& Index::_get_numerical_index() const {+ return numerical_index;+}diff --git a/test/art_test.cpp b/test/art_test.cppindex b843978a..e3ab87c9 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -1226,6 +1226,27 @@ TEST(ArtTest, test_search_negative_int64) {results.clear();}+TEST(ArtTest, test_search_negative_int64_large) {+ art_tree t;+ art_tree_init(&t);++ art_document doc = get_document(1);+ const int CHAR_LEN = 8;+ unsigned char chars[CHAR_LEN];++ const int64_t lmax = -1 * std::numeric_limits<std::int32_t>::max();++ encode_int64(-2, chars);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) chars, CHAR_LEN, &doc, 1));++ std::vector<const art_leaf *> results;++ int res = art_int64_search(&t, 1577836800, GREATER_THAN, results);+ //ASSERT_TRUE(res == 0);+ //ASSERT_EQ(0, results.size());+ //results.clear();+}+TEST(ArtTest, test_int32_array) {art_tree t;art_tree_init(&t);diff --git a/test/auth_manager_test.cpp b/test/auth_manager_test.cppindex de32fce8..8ecf45c3 100644--- a/test/auth_manager_test.cpp+++ b/test/auth_manager_test.cpp@@ -190,7 +190,16 @@ TEST_F(AuthManagerTest, VerifyAuthentication) {}TEST_F(AuthManagerTest, GenerationOfAPIAction) {+ std::map<int64_t, int64_t> int64map;+ int64map[1] = 1;+ int64map[2] = 2;+ int64map[3] = 3;++ std::cout << (int64map.lower_bound(5) == int64map.end());+ return ;+route_path rpath_search = route_path("GET", {"collections", ":collection", "documents", "search"}, nullptr, false, false);+ route_path rpath_coll_create = route_path("POST", {"collections"}, nullptr, false, false);route_path rpath_coll_get = route_path("GET", {"collections", ":collection"}, nullptr, false, false);route_path rpath_coll_list = route_path("GET", {"collections"}, nullptr, false, false);route_path rpath_keys_post = route_path("POST", {"keys"}, nullptr, false, false);@@ -198,6 +207,7 @@ TEST_F(AuthManagerTest, GenerationOfAPIAction) {route_path rpath_override_upsert = route_path("PUT", {"collections", ":collection", "overrides", ":id"}, nullptr, false, false);ASSERT_STREQ("documents:search", rpath_search._get_action().c_str());+ ASSERT_STREQ("collections:create", rpath_coll_create._get_action().c_str());ASSERT_STREQ("collections:get", rpath_coll_get._get_action().c_str());ASSERT_STREQ("collections:list", rpath_coll_list._get_action().c_str());ASSERT_STREQ("keys:create", rpath_keys_post._get_action().c_str());diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex 8e101cd3..b488ea16 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -306,5 +306,38 @@ TEST_F(CollectionSortingTest, ThreeSortFieldsLimit) {ASSERT_FALSE(res_op.ok());ASSERT_STREQ("Only upto 3 sort_by fields can be specified.", res_op.error().c_str());+ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionSortingTest, NegativeInt64Value) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("points", field_types::INT64, false),+ };++ coll1 = collectionManager.get_collection("coll1");+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();+ }++ nlohmann::json doc1;++ doc1["id"] = "100";+ doc1["title"] = "The quick brown fox";+ doc1["points"] = -2678400;++ coll1->add(doc1.dump());++ std::vector<sort_by> sort_fields_desc = {+ sort_by("points", "DESC")+ };++ query_fields = {"title"};+ auto res = coll1->search("*", query_fields, "points:>=1577836800", {}, sort_fields_desc, 0, 10, 1, FREQUENCY,+ false).get();++ ASSERT_EQ(0, res["found"].get<size_t>());+collectionManager.drop_collection("coll1");}\ No newline at end of filediff --git a/test/collection_test.cpp b/test/collection_test.cppindex 503bf0aa..63b2701b 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1772,18 +1772,20 @@ TEST_F(CollectionTest, DeletionOfDocumentArrayFields) {// also assert against the actual indexIndex *index = coll1->_get_indexes()[0]; // seq id will always be zero for first documentauto search_index = index->_get_search_index();+ auto numerical_index = index->_get_numerical_index();auto strarray_tree = search_index["strarray"];- auto int32array_tree = search_index["int32array"];- auto int64array_tree = search_index["int64array"];- auto floatarray_tree = search_index["floatarray"];- auto boolarray_tree = search_index["boolarray"];+ auto int32array_tree = numerical_index["int32array"];+ auto int64array_tree = numerical_index["int64array"];+ auto floatarray_tree = numerical_index["floatarray"];+ auto boolarray_tree = numerical_index["boolarray"];ASSERT_EQ(0, art_size(strarray_tree));- ASSERT_EQ(0, art_size(int32array_tree));- ASSERT_EQ(0, art_size(int64array_tree));- ASSERT_EQ(0, art_size(floatarray_tree));- ASSERT_EQ(0, art_size(boolarray_tree));++ ASSERT_EQ(0, int32array_tree->size());+ ASSERT_EQ(0, int64array_tree->size());+ ASSERT_EQ(0, floatarray_tree->size());+ ASSERT_EQ(0, boolarray_tree->size());collectionManager.drop_collection("coll1");}@@ -2005,6 +2007,8 @@ TEST_F(CollectionTest, SearchHighlightShouldUseHighlightTags) {ASSERT_STREQ("The quick brown fox jumped over the <em class=\"h\">lazy</em> fox. ",res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");}TEST_F(CollectionTest, SearchHighlightWithNewLine) {