Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fixed a bug with hlighlighting in upsert import.
Kishore Nallan9adbdd1576c0
diff --git a/include/array.h b/include/array.hindex cb402b96..4510a5bb 100644--- a/include/array.h+++ b/include/array.h@@ -18,6 +18,8 @@ private:}public:+ void load(const uint32_t *sorted_array, uint32_t array_length, uint32_t m, uint32_t M);+uint32_t at(uint32_t index);bool contains(uint32_t value);@@ -26,5 +28,7 @@ public:bool append(uint32_t value);+ bool insert(size_t index, const uint32_t* values, size_t num_values);+void remove_index(uint32_t start_index, uint32_t end_index);};\ No newline at end of filediff --git a/include/array_base.h b/include/array_base.hindex 450e0a0e..d4eae6a8 100644--- a/include/array_base.h+++ b/include/array_base.h@@ -36,7 +36,8 @@ public:in = nullptr;}- uint32_t* uncompress();+ // len determines length of output buffer (default: length of input)+ uint32_t* uncompress(uint32_t len=0);uint32_t getSizeInBytes();diff --git a/include/collection.h b/include/collection.hindex ca9dc32e..62dcd6c4 100644--- a/include/collection.h+++ b/include/collection.h@@ -308,6 +308,6 @@ public:void get_doc_changes(const nlohmann::json &document, nlohmann::json &old_doc,nlohmann::json &new_doc,- nlohmann::json &changed_doc);+ nlohmann::json &del_doc);};diff --git a/include/index.h b/include/index.hindex 9e767b2a..b8ad22e1 100644--- a/include/index.h+++ b/include/index.h@@ -92,7 +92,7 @@ struct index_record {nlohmann::json doc;nlohmann::json old_doc;nlohmann::json new_doc;- nlohmann::json changed_doc;+ nlohmann::json del_doc;index_operation_t operation;@@ -191,7 +191,7 @@ private:void index_bool_array_field(const std::vector<bool> & values, const int64_t score, art_tree *t, uint32_t seq_id) const;- void remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,+ void remove_and_shift_offset_index(sorted_array& offset_index, const uint32_t* indices_sorted,const uint32_t indices_length);uint32_t* collate_leaf_ids(const std::vector<const art_leaf *> &leaves, size_t& result_ids_len) const;@@ -304,5 +304,10 @@ public:void eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,const std::vector<art_leaf *> &query_suggestion,uint32_t *exact_strt_ids, size_t& exact_strt_size) const;++ void scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, nlohmann::json& old_doc);++ void tokenize_doc_field(const nlohmann::json& document, const std::string& field_name, const field& search_field,+ std::vector<std::string>& tokens);};diff --git a/include/sorted_array.h b/include/sorted_array.hindex 5df7037f..4e84166c 100644--- a/include/sorted_array.h+++ b/include/sorted_array.h@@ -8,6 +8,7 @@#include <limits>#include <iostream>#include "array_base.h"+#include "logger.h"class sorted_array: public array_base {private:@@ -16,7 +17,15 @@ private:uint32_t m = std::min(min, value);uint32_t M = std::max(max, value);uint32_t bnew = required_bits(M - m);- return METADATA_OVERHEAD + 4 + for_compressed_size_bits(new_length, bnew);+ uint32_t size_bits = for_compressed_size_bits(new_length, bnew);+++ /*if(new_length == 15) {+ LOG(INFO) << "value: " << value << ", m: " << m << ", M: " << M << ", bnew: "+ << bnew << ", size_bits: " << size_bits;+ }*/++ return METADATA_OVERHEAD + 4 + size_bits;}uint32_t lower_bound_search_bits(const uint8_t *in, uint32_t imin, uint32_t imax, uint32_t base,@@ -39,7 +48,9 @@ public:void indexOf(const uint32_t *values, const size_t values_len, uint32_t* indices);// returns false if malloc fails- bool append(uint32_t value);+ size_t append(uint32_t value);++ bool insert(size_t index, uint32_t value);- void remove_values(uint32_t *sorted_values, uint32_t values_length);+ void remove_values(uint32_t *sorted_values, uint32_t sorted_values_length);};\ No newline at end of filediff --git a/src/array.cpp b/src/array.cppindex 7a009d2c..b5805cf2 100644--- a/src/array.cpp+++ b/src/array.cpp@@ -41,6 +41,47 @@ bool array::append(uint32_t value) {return true;}+void array::load(const uint32_t *sorted_array, const uint32_t array_length, const uint32_t m, const uint32_t M) {+ min = m;+ max = M;++ uint32_t size_required = (uint32_t) (unsorted_append_size_required(max, array_length) * FOR_GROWTH_FACTOR);+ uint8_t *out = (uint8_t *) malloc(size_required * sizeof *out);+ uint32_t actual_size = for_compress_unsorted(sorted_array, out, array_length);++ free(in);+ in = nullptr;++ in = out;+ length = array_length;+ size_bytes = size_required;+ length_bytes = actual_size;+}++bool array::insert(size_t index, const uint32_t* values, size_t num_values) {+ if(index >= length) {+ return false;+ }++ uint32_t *curr_array = uncompress(length+num_values);+ memmove(&curr_array[index+num_values], &curr_array[index], sizeof(uint32_t)*(length-index));++ uint32_t m, M;++ for(size_t i=0; i<num_values; i++) {+ uint32_t value = values[i];+ if(value < m) m = value;+ if(value > M) M = value;+ curr_array[index+i] = value;+ }++ load(curr_array, length+num_values, m, M);++ delete [] curr_array;++ return true;+}+void array::remove_index(uint32_t start_index, uint32_t end_index) {uint32_t *curr_array = uncompress();diff --git a/src/array_base.cpp b/src/array_base.cppindex 4a9a29ff..2eae9dfd 100644--- a/src/array_base.cpp+++ b/src/array_base.cpp@@ -1,7 +1,8 @@#include "array_base.h"-uint32_t* array_base::uncompress() {- uint32_t *out = new uint32_t[length];+uint32_t* array_base::uncompress(uint32_t len) {+ uint32_t actual_len = std::max(len, length);+ uint32_t *out = new uint32_t[actual_len];for_uncompress(in, out, length);return out;}diff --git a/src/art.cpp b/src/art.cppindex 0cd2eed3..a77e60cd 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -39,6 +39,8 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *void art_int_fuzzy_recurse(art_node *n, int depth, const unsigned char* int_str, int int_str_len,NUM_COMPARATOR comparator, std::vector<const art_leaf *> &results);+static void insert_and_shift_offset_index(sorted_array& offset_index, const uint32_t index, const uint32_t num_offsets);+bool compare_art_leaf_frequency(const art_leaf *a, const art_leaf *b) {return a->values->ids.getLength() > b->values->ids.getLength();}@@ -408,13 +410,40 @@ art_leaf* art_maximum(art_tree *t) {static void add_document_to_leaf(const art_document *document, art_leaf *leaf) {leaf->max_score = MAX(leaf->max_score, document->score);- leaf->values->ids.append(document->id);- uint32_t curr_index = leaf->values->offsets.getLength();- leaf->values->offset_index.append(curr_index);+ size_t inserted_index = leaf->values->ids.append(document->id);++ if(inserted_index == leaf->values->ids.getLength()-1) {+ // treat as appends+ uint32_t curr_index = leaf->values->offsets.getLength();+ leaf->values->offset_index.append(curr_index);+ for(uint32_t i=0; i<document->offsets_len; i++) {+ leaf->values->offsets.append(document->offsets[i]);+ }+ } else {+ uint32_t existing_offset_index = leaf->values->offset_index.at(inserted_index);+ insert_and_shift_offset_index(leaf->values->offset_index, inserted_index, document->offsets_len);+ leaf->values->offsets.insert(existing_offset_index, document->offsets, document->offsets_len);+ }+}- for(uint32_t i=0; i<document->offsets_len; i++) {- leaf->values->offsets.append(document->offsets[i]);+void insert_and_shift_offset_index(sorted_array& offset_index, const uint32_t index, const uint32_t num_offsets) {+ uint32_t existing_offset_index = offset_index.at(index);+ uint32_t length = offset_index.getLength();+ uint32_t new_length = length + 1;+ uint32_t *curr_array = offset_index.uncompress(new_length);++ memmove(&curr_array[index+1], &curr_array[index], sizeof(uint32_t)*(length - index));+ curr_array[index] = existing_offset_index;++ uint32_t curr_index = index + 1;+ while(curr_index < new_length) {+ curr_array[curr_index] += num_offsets;+ curr_index++;}++ offset_index.load(curr_array, new_length);++ delete [] curr_array;}static art_leaf* make_leaf(const unsigned char *key, uint32_t key_len, art_document *document) {diff --git a/src/auth_manager.cpp b/src/auth_manager.cppindex 3df010b2..14a3f407 100644--- a/src/auth_manager.cpp+++ b/src/auth_manager.cpp@@ -5,7 +5,7 @@ constexpr const char* AuthManager::DOCUMENTS_SEARCH_ACTION;Option<bool> AuthManager::init(Store *store) {// This function must be idempotent, i.e. when called multiple times, must produce the same state without leaks- LOG(INFO) << "AuthManager::init()";+ //LOG(INFO) << "AuthManager::init()";this->store = store;@@ -157,7 +157,7 @@ bool AuthManager::authenticate(const std::string& req_api_key, const std::string}// enrich params with values from embedded_params- for (const auto& it: embedded_params.items()){+ for(auto it = embedded_params.begin(); it != embedded_params.end(); ++it) {if(params.count(it.key()) == 0) {params[it.key()] = it.value();} else if(it.key() == "filter_by") {diff --git a/src/collection.cpp b/src/collection.cppindex 9debd576..72f0728d 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -207,18 +207,17 @@ Option<nlohmann::json> Collection::add(const std::string & json_str, const bool// fetch original doc from store, create new doc and write it backnlohmann::json old_doc;nlohmann::json new_doc;- nlohmann::json changed_doc; // to identify changed fields+ nlohmann::json del_doc; // to identify fields that should be potentially removedget_document_from_store(get_seq_id_key(seq_id), old_doc);- get_doc_changes(document, old_doc, new_doc, changed_doc);+ get_doc_changes(document, old_doc, new_doc, del_doc);- //LOG(INFO) << "changed_doc: " << changed_doc;-- remove_document(changed_doc, seq_id, false);+ //LOG(INFO) << "del_doc: " << del_doc;+ remove_document(del_doc, seq_id, false);const Option<uint32_t> & index_memory_op = index_in_memory(document, seq_id, is_update);if(!index_memory_op.ok()) {- index_in_memory(changed_doc, seq_id, true);+ index_in_memory(del_doc, seq_id, true);return Option<nlohmann::json>(index_memory_op.code(), index_memory_op.error());}@@ -256,16 +255,17 @@ Option<nlohmann::json> Collection::add(const std::string & json_str, const bool}void Collection::get_doc_changes(const nlohmann::json &document, nlohmann::json &old_doc,- nlohmann::json &new_doc, nlohmann::json &changed_doc) {+ nlohmann::json &new_doc, nlohmann::json &del_doc) {- for(const auto& it: old_doc.items()) {+ for(auto it = old_doc.begin(); it != old_doc.end(); ++it) {new_doc[it.key()] = it.value();}- for(const auto& it: document.items()) {+ for(auto it = document.begin(); it != document.end(); ++it) {new_doc[it.key()] = it.value();if(old_doc.count(it.key()) != 0) {- changed_doc[it.key()] = old_doc[it.key()];+ // key exists in the stored doc, so it must be reindexed+ del_doc[it.key()] = old_doc[it.key()];}}}@@ -302,7 +302,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, constif(is_update) {record.operation = UPDATE;get_document_from_store(get_seq_id_key(seq_id), record.old_doc);- get_doc_changes(document, record.old_doc, record.new_doc, record.changed_doc);+ get_doc_changes(document, record.old_doc, record.new_doc, record.del_doc);}}diff --git a/src/http_server.cpp b/src/http_server.cppindex 75b07790..11e750ea 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -129,6 +129,7 @@ int HttpServer::create_listener() {ctx.globalconf->server_name = h2o_strdup(nullptr, "", SIZE_MAX);ctx.globalconf->http2.active_stream_window_size = ACTIVE_STREAM_WINDOW_SIZE;ctx.globalconf->http2.idle_timeout = REQ_TIMEOUT_MS;+ ctx.globalconf->max_request_entity_size = (1024 * 1024 * 1024); // 1 GBctx.globalconf->http1.req_timeout = REQ_TIMEOUT_MS;ctx.globalconf->http1.req_io_timeout = REQ_TIMEOUT_MS;diff --git a/src/index.cpp b/src/index.cppindex 5721672e..8bf15ec2 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -321,6 +321,43 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,return Option<>(200);}+void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, nlohmann::json& old_doc) {+ for (auto it = del_doc.begin(); it != del_doc.end(); ++it) {+ const std::string& field_name = it.key();+ const auto& search_field_it = search_schema.find(field_name);+ if(search_field_it == search_schema.end()) {+ continue;+ }++ const auto& search_field = search_field_it->second;++ // Go through all the field names and find the keys+values so that they can be removed from in-memory index+ std::vector<std::string> reindex_tokens;+ std::vector<std::string> old_tokens;+ tokenize_doc_field(update_doc, field_name, search_field, reindex_tokens);+ tokenize_doc_field(old_doc, field_name, search_field, old_tokens);++ if(old_tokens.size() != reindex_tokens.size()) {+ continue;+ }++ bool exact_match = true;++ for(size_t i=0; i<reindex_tokens.size(); i++) {+ const std::string& reindex_val = reindex_tokens[i];+ const std::string& old_val = old_tokens[i];+ if(reindex_val != old_val) {+ exact_match = false;+ break;+ }+ }++ if(exact_match) {+ del_doc.erase(field_name);+ }+ }+}+size_t Index::batch_memory_index(Index *index, std::vector<index_record> & iter_batch,const std::string & default_sorting_field,const std::unordered_map<std::string, field> & search_schema,@@ -347,19 +384,24 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record> & iter_}if(is_update) {- index->remove(index_rec.seq_id, index_rec.changed_doc);+ // scrub string fields to reduce delete ops+ index->scrub_reindex_doc(index_rec.doc, index_rec.del_doc, index_rec.old_doc);+ index->remove(index_rec.seq_id, index_rec.del_doc);}Option<uint32_t> index_mem_op = index->index_in_memory(index_rec.doc, index_rec.seq_id,default_sorting_field, is_update);if(!index_mem_op.ok()) {- index->index_in_memory(index_rec.changed_doc, index_rec.seq_id, default_sorting_field, true);+ index->index_in_memory(index_rec.del_doc, index_rec.seq_id, default_sorting_field, true);index_rec.index_failure(index_mem_op.code(), index_mem_op.error());continue;}index_rec.index_success();- num_indexed++;++ if(!is_update) {+ num_indexed++;+ }}}@@ -391,6 +433,7 @@ void Index::insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,art_doc.offsets[i] = kv.second[i];}+ //LOG(INFO) << "key: " << key << ", art_doc.id: " << art_doc.id;art_insert(t, key, key_len, &art_doc, num_hits);delete [] art_doc.offsets;art_doc.offsets = nullptr;@@ -528,6 +571,10 @@ void Index::index_string_field(const std::string & text, const int64_t score, artoken_to_offsets[token].push_back(i);}+ /*if(seq_id == 0) {+ LOG(INFO) << "field name: " << a_field.name;+ }*/+insert_doc(score, t, seq_id, token_to_offsets);if(facet_id >= 0) {@@ -1734,6 +1781,11 @@ void Index::populate_token_positions(const std::vector<art_leaf *>& query_sugges// a) last element is array_index b) second and third last elements will be largest offset// (last element is repeated to indicate end of offsets for a given array index)+ /*uint32_t* offsets = token_leaf->values->offsets.uncompress();+ for(size_t ii=0; ii < token_leaf->values->offsets.getLength(); ii++) {+ LOG(INFO) << "offset: " << offsets[ii];+ }*/+uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?token_leaf->values->offsets.getLength() :@@ -1789,8 +1841,8 @@ inline std::vector<art_leaf *> Index::next_suggestion(const std::vector<token_careturn query_suggestion;}-void Index::remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,- const uint32_t indices_length) {+void Index::remove_and_shift_offset_index(sorted_array& offset_index, const uint32_t* indices_sorted,+ const uint32_t indices_length) {uint32_t *curr_array = offset_index.uncompress();uint32_t *new_array = new uint32_t[offset_index.getLength()];@@ -1826,85 +1878,24 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & docstd::unordered_map<std::string, size_t> facet_to_index;get_facet_to_index(facet_to_index);- for(const auto& it: document.items()) {+ for(auto it = document.begin(); it != document.end(); ++it) {const std::string& field_name = it.key();- const auto& search_field = search_schema.find(field_name);- if(search_field == search_schema.end()) {+ const auto& search_field_it = search_schema.find(field_name);+ if(search_field_it == search_schema.end()) {continue;}+ const auto& search_field = search_field_it->second;+// Go through all the field names and find the keys+values so that they can be removed from in-memory indexstd::vector<std::string> tokens;- if(search_field->second.type == field_types::STRING) {- StringUtils::split(document[search_field->first], tokens, " ");- } else if(search_field->second.type == field_types::STRING_ARRAY) {- std::vector<std::string> values = document[search_field->first].get<std::vector<std::string>>();- for(const std::string & value: values) {- StringUtils::split(value, tokens, " ");- }- } else if(search_field->second.type == field_types::INT32) {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];- int32_t value = document[search_field->first].get<int32_t>();- encode_int32(value, key);- tokens.emplace_back((char*)key, KEY_LEN);- } else if(search_field->second.type == field_types::INT32_ARRAY) {- std::vector<int32_t> values = document[search_field->first].get<std::vector<int32_t>>();- for(const int32_t value: values) {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];- encode_int32(value, key);- tokens.emplace_back((char*)key, KEY_LEN);- }- } else if(search_field->second.type == field_types::INT64) {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];- int64_t value = document[search_field->first].get<int64_t>();- encode_int64(value, key);- tokens.emplace_back((char*)key, KEY_LEN);- } else if(search_field->second.type == field_types::INT64_ARRAY) {- std::vector<int64_t> values = document[search_field->first].get<std::vector<int64_t>>();- for(const int64_t value: values) {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];- encode_int64(value, key);- tokens.emplace_back((char*)key, KEY_LEN);- }- } else if(search_field->second.type == field_types::FLOAT) {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];- int64_t value = document[search_field->first].get<int64_t>();- encode_float(value, key);- tokens.emplace_back((char*)key, KEY_LEN);- } else if(search_field->second.type == field_types::FLOAT_ARRAY) {- std::vector<float> values = document[search_field->first].get<std::vector<float>>();- for(const float value: values) {- const int KEY_LEN = 8;- unsigned char key[KEY_LEN];- encode_float(value, key);- tokens.emplace_back((char*)key, KEY_LEN);- }- } else if(search_field->second.type == field_types::BOOL) {- const int KEY_LEN = 1;- unsigned char key[KEY_LEN];- bool value = document[search_field->first].get<bool>();- key[0] = value ? '1' : '0';- tokens.emplace_back((char*)key, KEY_LEN);- } else if(search_field->second.type == field_types::BOOL_ARRAY) {- std::vector<bool> values = document[search_field->first].get<std::vector<bool>>();- for(const bool value: values) {- const int KEY_LEN = 1;- unsigned char key[KEY_LEN];- key[0] = value ? '1' : '0';- tokens.emplace_back((char*)key, KEY_LEN);- }- }+ tokenize_doc_field(document, field_name, search_field, tokens);for(auto & token: tokens) {const unsigned char *key;int key_len;- if(search_field->second.type == field_types::STRING_ARRAY || search_field->second.type == field_types::STRING) {+ if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {string_utils.unicode_normalize(token);key = (const unsigned char *) token.c_str();key_len = (int) (token.length() + 1);@@ -1913,7 +1904,7 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & dockey_len = (int) (token.length());}- art_leaf* leaf = (art_leaf *) art_search(search_index.at(search_field->first), key, key_len);+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name), key, key_len);if(leaf != nullptr) {uint32_t seq_id_values[1] = {seq_id};uint32_t doc_index = leaf->values->ids.indexOf(seq_id);@@ -1941,7 +1932,7 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & docLOG(INFO) << "----";*/if(leaf->values->ids.getLength() == 0) {- art_values* values = (art_values*) art_delete(search_index.at(search_field->first), key, key_len);+ art_values* values = (art_values*) art_delete(search_index.at(field_name), key, key_len);delete values;}}@@ -1963,6 +1954,74 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & docreturn Option<uint32_t>(seq_id);}+void Index::tokenize_doc_field(const nlohmann::json& document, const std::string& field_name, const field& search_field,+ std::vector<std::string>& tokens) {+ if(search_field.type == field_types::STRING) {+ StringUtils::split(document[field_name], tokens, " ");+ } else if(search_field.type == field_types::STRING_ARRAY) {+ const std::vector<std::string>& values = document[field_name].get<std::vector<std::string>>();+ for(const std::string & value: values) {+ StringUtils::split(value, tokens, " ");+ }+ } else if(search_field.type == field_types::INT32) {+ const int KEY_LEN = 8;+ unsigned char key[KEY_LEN];+ const int32_t& value = document[field_name].get<int32_t>();+ encode_int32(value, key);+ tokens.emplace_back((char*)key, KEY_LEN);+ } else if(search_field.type == field_types::INT32_ARRAY) {+ const std::vector<int32_t>& values = document[field_name].get<std::vector<int32_t>>();+ for(const int32_t value: values) {+ const int KEY_LEN = 8;+ unsigned char key[KEY_LEN];+ encode_int32(value, key);+ tokens.emplace_back((char*)key, KEY_LEN);+ }+ } else if(search_field.type == field_types::INT64) {+ const int KEY_LEN = 8;+ unsigned char key[KEY_LEN];+ const int64_t& value = document[field_name].get<int64_t>();+ encode_int64(value, key);+ tokens.emplace_back((char*)key, KEY_LEN);+ } else if(search_field.type == field_types::INT64_ARRAY) {+ const std::vector<int64_t>& values = document[field_name].get<std::vector<int64_t>>();+ for(const int64_t value: values) {+ const int KEY_LEN = 8;+ unsigned char key[KEY_LEN];+ encode_int64(value, key);+ tokens.emplace_back((char*)key, KEY_LEN);+ }+ } else if(search_field.type == field_types::FLOAT) {+ const int KEY_LEN = 8;+ unsigned char key[KEY_LEN];+ const int64_t& value = document[field_name].get<int64_t>();+ encode_float(value, key);+ tokens.emplace_back((char*)key, KEY_LEN);+ } else if(search_field.type == field_types::FLOAT_ARRAY) {+ const std::vector<float>& values = document[field_name].get<std::vector<float>>();+ for(const float value: values) {+ const int KEY_LEN = 8;+ unsigned char key[KEY_LEN];+ encode_float(value, key);+ tokens.emplace_back((char*)key, KEY_LEN);+ }+ } else if(search_field.type == field_types::BOOL) {+ const int KEY_LEN = 1;+ unsigned char key[KEY_LEN];+ const bool& value = document[field_name].get<bool>();+ key[0] = value ? '1' : '0';+ tokens.emplace_back((char*)key, KEY_LEN);+ } else if(search_field.type == field_types::BOOL_ARRAY) {+ const std::vector<bool>& values = document[field_name].get<std::vector<bool>>();+ for(const bool value: values) {+ const int KEY_LEN = 1;+ unsigned char key[KEY_LEN];+ key[0] = value ? '1' : '0';+ tokens.emplace_back((char*)key, KEY_LEN);+ }+ }+}+art_leaf* Index::get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len) {const art_tree *t = search_index.at(field_name);return (art_leaf*) art_search(t, token, (int) token_len);diff --git a/src/sorted_array.cpp b/src/sorted_array.cppindex 60f0a962..9dadc51c 100644--- a/src/sorted_array.cpp+++ b/src/sorted_array.cpp@@ -1,5 +1,6 @@#include "sorted_array.h"#include "array_utils.h"+#include "logger.h"void sorted_array::load(const uint32_t *sorted_array, const uint32_t array_length) {min = array_length != 0 ? sorted_array[0] : 0;@@ -18,28 +19,82 @@ void sorted_array::load(const uint32_t *sorted_array, const uint32_t array_lengtlength_bytes = actual_size;}-bool sorted_array::append(uint32_t value) {+size_t sorted_array::append(uint32_t value) {uint32_t size_required = sorted_append_size_required(value, length+1);+ size_t min_expected_size = size_required + FOR_ELE_SIZE;- if(size_required+FOR_ELE_SIZE > size_bytes) {- // grow the array first- size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);- uint8_t *new_location = (uint8_t *) realloc(in, new_size);- if(new_location == NULL) {- abort();+ /*LOG(INFO) << "value: " << value << ", new_length: " << length+1 << ", size_bytes: " << size_bytes+ << ", size_required: " << size_required << ", min_expected_size: " << min_expected_size;++ if(value == 0 && size_bytes == 28) {+ uint32_t* arr = uncompress();+ for(size_t i=0; i<length; i++) {+ LOG(INFO) << "ele: " << arr[i];+ }++ delete [] arr;+ }*/++ if(value < max) {+ // we will have to re-encode the whole sequence again+ uint32_t* arr = uncompress(length+1);+ size_t i = 0;+ while(i < length+1) {+ if(value < arr[i]) {+ break;+ }+ i++;+ }++ for(size_t j=length; j>i; j--) {+ arr[j] = arr[j-1];+ }++ arr[i] = value;++ load(arr, length+1);+ delete [] arr;++ return i;+ } else {+ if(size_bytes < min_expected_size) {+ // grow the array first+ size_t new_size = min_expected_size * FOR_GROWTH_FACTOR;+ uint8_t *new_location = (uint8_t *) realloc(in, new_size);+ if(new_location == NULL) {+ abort();+ }+ in = new_location;+ size_bytes = (uint32_t) new_size;++ //LOG(INFO) << "new_size: " << new_size;}- in = new_location;- size_bytes = (uint32_t) new_size;++ uint32_t new_length_bytes = for_append_sorted(in, length, value);+ if(new_length_bytes == 0) return false;++ length_bytes = new_length_bytes;+ length++;++ if(value < min) min = value;+ if(value > max) max = value;++ return length-1;+ }+}++bool sorted_array::insert(size_t index, uint32_t value) {+ if(index >= length) {+ return false;}- uint32_t new_length_bytes = for_append_sorted(in, length, value);- if(new_length_bytes == 0) return false;+ uint32_t *curr_array = uncompress(length+1);+ memmove(&curr_array[index+1], &curr_array[index], sizeof(uint32_t)*(length-index));+ curr_array[index] = value;- length_bytes = new_length_bytes;- length++;+ load(curr_array, length+1);- if(value < min) min = value;- if(value > max) max = value;+ delete [] curr_array;return true;}@@ -150,20 +205,18 @@ void sorted_array::indexOf(const uint32_t *values, const size_t values_len, uintbinary_search_indices(values, head, tail, low_index, high_index, base, bits, indices);}-void sorted_array::remove_values(uint32_t *sorted_values, uint32_t values_length) {+void sorted_array::remove_values(uint32_t *sorted_values, uint32_t sorted_values_length) {uint32_t *curr_array = uncompress();uint32_t *new_array = new uint32_t[length];uint32_t new_index = 0;- uint32_t curr_index = 0;+uint32_t sorted_values_index = 0;+ uint32_t curr_index = 0;while(curr_index < length) {- if(sorted_values_index < values_length && curr_array[curr_index] >= sorted_values[sorted_values_index]) {- // skip copying- if(curr_array[curr_index] == sorted_values[sorted_values_index]) {- curr_index++;- }+ if(sorted_values_index < sorted_values_length && sorted_values[sorted_values_index] == curr_array[curr_index]) {+ curr_index++;sorted_values_index++;} else {new_array[new_index++] = curr_array[curr_index++];diff --git a/test/array_test.cpp b/test/array_test.cppindex 1efa0bd2..d59a7825 100644--- a/test/array_test.cpp+++ b/test/array_test.cpp@@ -45,6 +45,31 @@ TEST(ArrayTest, Append) {}}+TEST(ArrayTest, InsertValues) {+ std::vector<uint32_t> eles = {10, 1, 4, 5, 7};+ array arr;++ for(size_t i=0; i < eles.size(); i++) {+ arr.append(eles[i]);+ }++ uint32_t insert_arr[2] = {2, 3};+ arr.insert(2, insert_arr, 2);+ eles = {10, 1, 2, 3, 4, 5, 7};++ for(size_t i=0; i < eles.size(); i++) {+ ASSERT_EQ(eles[i], arr.at(i));+ }++ uint32_t insert_arr2[2] = {20, 25};+ arr.insert(6, insert_arr2, 2);++ eles = {10, 1, 2, 3, 4, 5, 20, 25, 7};+ for(size_t i=0; i < eles.size(); i++) {+ ASSERT_EQ(eles[i], arr.at(i));+ }+}+TEST(ArrayTest, Uncompress) {const size_t SIZE = 10*1000;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 7a7537a2..d8d618f8 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1316,7 +1316,7 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {coll_mul_fields = collectionManager.get_collection("coll_mul_fields");if(coll_mul_fields == nullptr) {- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 1, fields, "points").get();}// try importing records@@ -1326,8 +1326,8 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {ASSERT_EQ(18, import_response["num_imported"].get<int>());// import some new records along with updates- std::vector<std::string> more_records = {R"({"id": "0", "title": "Wake up, Harry"})",- R"({"id": "2", "cast": ["Kim Werrel", "Random Wake"]})",+ std::vector<std::string> more_records = {R"({"id": "0", "title": "The Fifth Harry"})",+ R"({"id": "2", "cast": ["Chris Fisher", "Rand Alan"]})",R"({"id": "18", "title": "Back Again Forest", "points": 45, "starring": "Ronald Wells", "cast": ["Dant Saren"]})",R"({"id": "6", "points": 77})"};@@ -1344,7 +1344,18 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {ASSERT_EQ(1, import_results[i].size());}- auto results = coll_mul_fields->search("burgundy", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ auto results = coll_mul_fields->search("*", query_fields, "", {}, sort_fields, 0, 30, 1, FREQUENCY, false).get();+ ASSERT_EQ(19, results["hits"].size());++ ASSERT_EQ(19, coll_mul_fields->get_num_documents());++ results = coll_mul_fields->search("fifth", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(2, results["hits"].size());++ ASSERT_STREQ("The <mark>Fifth</mark> Harry", results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());+ ASSERT_STREQ("The Woman in the <mark>Fifth</mark> from Kristin", results["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());++ results = coll_mul_fields->search("burgundy", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(0, results["hits"].size());results = coll_mul_fields->search("harry", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();@@ -1385,6 +1396,18 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {ASSERT_FALSE(import_results[1]["success"].get<bool>());ASSERT_STREQ("A document with id 1 already exists.", import_results[0]["error"].get<std::string>().c_str());ASSERT_STREQ("A document with id 5 already exists.", import_results[1]["error"].get<std::string>().c_str());++ // update document with verbatim fields, except for points+ more_records = {R"({"id": "3", "cast":["Matt Damon","Ben Affleck","Minnie Driver"],+ "points":70,"starring":"Robin Williams","starring_facet":"Robin Williams",+ "title":"Good Will Hunting"})"};++ import_response = coll_mul_fields->add_many(more_records, true);+ ASSERT_TRUE(import_response["success"].get<bool>());+ ASSERT_EQ(1, import_response["num_imported"].get<int>());++ results = coll_mul_fields->search("Good Will Hunting", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(70, results["hits"][0]["document"]["points"].get<uint32_t>());}TEST_F(CollectionTest, ImportDocuments) {@@ -2356,6 +2379,8 @@ TEST_F(CollectionTest, UpdateDocument) {add_op = coll1->add(doc.dump(), true);ASSERT_TRUE(add_op.ok());+ ASSERT_EQ(1, coll1->get_num_documents());+res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();diff --git a/test/sorted_array_test.cpp b/test/sorted_array_test.cppindex dfdb9dcc..93351649 100644--- a/test/sorted_array_test.cpp+++ b/test/sorted_array_test.cpp@@ -33,6 +33,72 @@ TEST(SortedArrayTest, Append) {EXPECT_EQ(arr_small.at(0), 100);}+TEST(SortedArrayTest, AppendOutOfOrder) {+ sorted_array arr;+ for(size_t i=5; i<=10; i++) {+ arr.append(i);+ }++ EXPECT_EQ(6, arr.getLength());++ arr.append(1);+ arr.append(3);+ arr.append(2);+ arr.append(4);+ arr.append(11);+ arr.append(14);+ arr.append(12);++ EXPECT_EQ(13, arr.getLength());+}++TEST(SortedArrayTest, InsertAtIndex) {+ std::vector<uint32_t> eles;+ sorted_array arr;+ for(size_t i=5; i<=9; i++) {+ arr.append(i);+ }++ arr.append(11);+ eles = {5, 6, 7, 8, 9, 11};++ for(size_t i=0; i < eles.size(); i++) {+ ASSERT_EQ(eles[i], arr.at(i));+ }++ arr.insert(0, 1);+ eles = { 1, 5, 6, 7, 8, 9, 11 };++ for(size_t i=0; i < eles.size(); i++) {+ ASSERT_EQ(eles[i], arr.at(i));+ }++ ASSERT_EQ(1, arr.at(0));+ ASSERT_EQ(5, arr.at(1));++ arr.insert(1, 2);+ eles = {1, 2, 5, 6, 7, 8, 9, 11};+ ASSERT_EQ(1, arr.at(0));+ ASSERT_EQ(2, arr.at(1));+ ASSERT_EQ(8, arr.getLength());++ for(size_t i=0; i < eles.size(); i++) {+ ASSERT_EQ(eles[i], arr.at(i));+ }++ arr.insert(7, 10);+ eles = { 1, 2, 5, 6, 7, 8, 9, 10, 11};+ ASSERT_EQ(10, arr.at(7));+ ASSERT_EQ(11, arr.at(8));+ ASSERT_EQ(9, arr.getLength());++ for(size_t i=0; i < eles.size(); i++) {+ ASSERT_EQ(eles[i], arr.at(i));+ }++ ASSERT_FALSE(arr.insert(9, 12)); // index out of range+}+TEST(SortedArrayTest, Load) {sorted_array arr;