Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fixed a bug with hlighlighting in upsert import.

Kishore Nallan9adbdd1576c0
diff --git a/include/array.h b/include/array.h
index cb402b96..4510a5bb 100644
--- a/include/array.h
+++ b/include/array.h
@@ -18,6 +18,8 @@ private:
}
public:
+ void load(const uint32_t *sorted_array, uint32_t array_length, uint32_t m, uint32_t M);
+
uint32_t at(uint32_t index);
bool contains(uint32_t value);
@@ -26,5 +28,7 @@ public:
bool append(uint32_t value);
+ bool insert(size_t index, const uint32_t* values, size_t num_values);
+
void remove_index(uint32_t start_index, uint32_t end_index);
};
\ No newline at end of file
diff --git a/include/array_base.h b/include/array_base.h
index 450e0a0e..d4eae6a8 100644
--- a/include/array_base.h
+++ b/include/array_base.h
@@ -36,7 +36,8 @@ public:
in = nullptr;
}
- uint32_t* uncompress();
+ // len determines length of output buffer (default: length of input)
+ uint32_t* uncompress(uint32_t len=0);
uint32_t getSizeInBytes();
diff --git a/include/collection.h b/include/collection.h
index ca9dc32e..62dcd6c4 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -308,6 +308,6 @@ public:
void get_doc_changes(const nlohmann::json &document, nlohmann::json &old_doc,
nlohmann::json &new_doc,
- nlohmann::json &changed_doc);
+ nlohmann::json &del_doc);
};
diff --git a/include/index.h b/include/index.h
index 9e767b2a..b8ad22e1 100644
--- a/include/index.h
+++ b/include/index.h
@@ -92,7 +92,7 @@ struct index_record {
nlohmann::json doc;
nlohmann::json old_doc;
nlohmann::json new_doc;
- nlohmann::json changed_doc;
+ nlohmann::json del_doc;
index_operation_t operation;
@@ -191,7 +191,7 @@ private:
void index_bool_array_field(const std::vector<bool> & values, const int64_t score, art_tree *t, uint32_t seq_id) const;
- void remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,
+ void remove_and_shift_offset_index(sorted_array& offset_index, const uint32_t* indices_sorted,
const uint32_t indices_length);
uint32_t* collate_leaf_ids(const std::vector<const art_leaf *> &leaves, size_t& result_ids_len) const;
@@ -304,5 +304,10 @@ public:
void eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,
const std::vector<art_leaf *> &query_suggestion,
uint32_t *exact_strt_ids, size_t& exact_strt_size) const;
+
+ void scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, nlohmann::json& old_doc);
+
+ void tokenize_doc_field(const nlohmann::json& document, const std::string& field_name, const field& search_field,
+ std::vector<std::string>& tokens);
};
diff --git a/include/sorted_array.h b/include/sorted_array.h
index 5df7037f..4e84166c 100644
--- a/include/sorted_array.h
+++ b/include/sorted_array.h
@@ -8,6 +8,7 @@
#include <limits>
#include <iostream>
#include "array_base.h"
+#include "logger.h"
class sorted_array: public array_base {
private:
@@ -16,7 +17,15 @@ private:
uint32_t m = std::min(min, value);
uint32_t M = std::max(max, value);
uint32_t bnew = required_bits(M - m);
- return METADATA_OVERHEAD + 4 + for_compressed_size_bits(new_length, bnew);
+ uint32_t size_bits = for_compressed_size_bits(new_length, bnew);
+
+
+ /*if(new_length == 15) {
+ LOG(INFO) << "value: " << value << ", m: " << m << ", M: " << M << ", bnew: "
+ << bnew << ", size_bits: " << size_bits;
+ }*/
+
+ return METADATA_OVERHEAD + 4 + size_bits;
}
uint32_t lower_bound_search_bits(const uint8_t *in, uint32_t imin, uint32_t imax, uint32_t base,
@@ -39,7 +48,9 @@ public:
void indexOf(const uint32_t *values, const size_t values_len, uint32_t* indices);
// returns false if malloc fails
- bool append(uint32_t value);
+ size_t append(uint32_t value);
+
+ bool insert(size_t index, uint32_t value);
- void remove_values(uint32_t *sorted_values, uint32_t values_length);
+ void remove_values(uint32_t *sorted_values, uint32_t sorted_values_length);
};
\ No newline at end of file
diff --git a/src/array.cpp b/src/array.cpp
index 7a009d2c..b5805cf2 100644
--- a/src/array.cpp
+++ b/src/array.cpp
@@ -41,6 +41,47 @@ bool array::append(uint32_t value) {
return true;
}
+void array::load(const uint32_t *sorted_array, const uint32_t array_length, const uint32_t m, const uint32_t M) {
+ min = m;
+ max = M;
+
+ uint32_t size_required = (uint32_t) (unsorted_append_size_required(max, array_length) * FOR_GROWTH_FACTOR);
+ uint8_t *out = (uint8_t *) malloc(size_required * sizeof *out);
+ uint32_t actual_size = for_compress_unsorted(sorted_array, out, array_length);
+
+ free(in);
+ in = nullptr;
+
+ in = out;
+ length = array_length;
+ size_bytes = size_required;
+ length_bytes = actual_size;
+}
+
+bool array::insert(size_t index, const uint32_t* values, size_t num_values) {
+ if(index >= length) {
+ return false;
+ }
+
+ uint32_t *curr_array = uncompress(length+num_values);
+ memmove(&curr_array[index+num_values], &curr_array[index], sizeof(uint32_t)*(length-index));
+
+ uint32_t m, M;
+
+ for(size_t i=0; i<num_values; i++) {
+ uint32_t value = values[i];
+ if(value < m) m = value;
+ if(value > M) M = value;
+ curr_array[index+i] = value;
+ }
+
+ load(curr_array, length+num_values, m, M);
+
+ delete [] curr_array;
+
+ return true;
+}
+
void array::remove_index(uint32_t start_index, uint32_t end_index) {
uint32_t *curr_array = uncompress();
diff --git a/src/array_base.cpp b/src/array_base.cpp
index 4a9a29ff..2eae9dfd 100644
--- a/src/array_base.cpp
+++ b/src/array_base.cpp
@@ -1,7 +1,8 @@
#include "array_base.h"
-uint32_t* array_base::uncompress() {
- uint32_t *out = new uint32_t[length];
+uint32_t* array_base::uncompress(uint32_t len) {
+ uint32_t actual_len = std::max(len, length);
+ uint32_t *out = new uint32_t[actual_len];
for_uncompress(in, out, length);
return out;
}
diff --git a/src/art.cpp b/src/art.cpp
index 0cd2eed3..a77e60cd 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -39,6 +39,8 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
void art_int_fuzzy_recurse(art_node *n, int depth, const unsigned char* int_str, int int_str_len,
NUM_COMPARATOR comparator, std::vector<const art_leaf *> &results);
+static void insert_and_shift_offset_index(sorted_array& offset_index, const uint32_t index, const uint32_t num_offsets);
+
bool compare_art_leaf_frequency(const art_leaf *a, const art_leaf *b) {
return a->values->ids.getLength() > b->values->ids.getLength();
}
@@ -408,13 +410,40 @@ art_leaf* art_maximum(art_tree *t) {
static void add_document_to_leaf(const art_document *document, art_leaf *leaf) {
leaf->max_score = MAX(leaf->max_score, document->score);
- leaf->values->ids.append(document->id);
- uint32_t curr_index = leaf->values->offsets.getLength();
- leaf->values->offset_index.append(curr_index);
+ size_t inserted_index = leaf->values->ids.append(document->id);
+
+ if(inserted_index == leaf->values->ids.getLength()-1) {
+ // treat as appends
+ uint32_t curr_index = leaf->values->offsets.getLength();
+ leaf->values->offset_index.append(curr_index);
+ for(uint32_t i=0; i<document->offsets_len; i++) {
+ leaf->values->offsets.append(document->offsets[i]);
+ }
+ } else {
+ uint32_t existing_offset_index = leaf->values->offset_index.at(inserted_index);
+ insert_and_shift_offset_index(leaf->values->offset_index, inserted_index, document->offsets_len);
+ leaf->values->offsets.insert(existing_offset_index, document->offsets, document->offsets_len);
+ }
+}
- for(uint32_t i=0; i<document->offsets_len; i++) {
- leaf->values->offsets.append(document->offsets[i]);
+void insert_and_shift_offset_index(sorted_array& offset_index, const uint32_t index, const uint32_t num_offsets) {
+ uint32_t existing_offset_index = offset_index.at(index);
+ uint32_t length = offset_index.getLength();
+ uint32_t new_length = length + 1;
+ uint32_t *curr_array = offset_index.uncompress(new_length);
+
+ memmove(&curr_array[index+1], &curr_array[index], sizeof(uint32_t)*(length - index));
+ curr_array[index] = existing_offset_index;
+
+ uint32_t curr_index = index + 1;
+ while(curr_index < new_length) {
+ curr_array[curr_index] += num_offsets;
+ curr_index++;
}
+
+ offset_index.load(curr_array, new_length);
+
+ delete [] curr_array;
}
static art_leaf* make_leaf(const unsigned char *key, uint32_t key_len, art_document *document) {
diff --git a/src/auth_manager.cpp b/src/auth_manager.cpp
index 3df010b2..14a3f407 100644
--- a/src/auth_manager.cpp
+++ b/src/auth_manager.cpp
@@ -5,7 +5,7 @@ constexpr const char* AuthManager::DOCUMENTS_SEARCH_ACTION;
Option<bool> AuthManager::init(Store *store) {
// This function must be idempotent, i.e. when called multiple times, must produce the same state without leaks
- LOG(INFO) << "AuthManager::init()";
+ //LOG(INFO) << "AuthManager::init()";
this->store = store;
@@ -157,7 +157,7 @@ bool AuthManager::authenticate(const std::string& req_api_key, const std::string
}
// enrich params with values from embedded_params
- for (const auto& it: embedded_params.items()){
+ for(auto it = embedded_params.begin(); it != embedded_params.end(); ++it) {
if(params.count(it.key()) == 0) {
params[it.key()] = it.value();
} else if(it.key() == "filter_by") {
diff --git a/src/collection.cpp b/src/collection.cpp
index 9debd576..72f0728d 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -207,18 +207,17 @@ Option<nlohmann::json> Collection::add(const std::string & json_str, const bool
// fetch original doc from store, create new doc and write it back
nlohmann::json old_doc;
nlohmann::json new_doc;
- nlohmann::json changed_doc; // to identify changed fields
+ nlohmann::json del_doc; // to identify fields that should be potentially removed
get_document_from_store(get_seq_id_key(seq_id), old_doc);
- get_doc_changes(document, old_doc, new_doc, changed_doc);
+ get_doc_changes(document, old_doc, new_doc, del_doc);
- //LOG(INFO) << "changed_doc: " << changed_doc;
-
- remove_document(changed_doc, seq_id, false);
+ //LOG(INFO) << "del_doc: " << del_doc;
+ remove_document(del_doc, seq_id, false);
const Option<uint32_t> & index_memory_op = index_in_memory(document, seq_id, is_update);
if(!index_memory_op.ok()) {
- index_in_memory(changed_doc, seq_id, true);
+ index_in_memory(del_doc, seq_id, true);
return Option<nlohmann::json>(index_memory_op.code(), index_memory_op.error());
}
@@ -256,16 +255,17 @@ Option<nlohmann::json> Collection::add(const std::string & json_str, const bool
}
void Collection::get_doc_changes(const nlohmann::json &document, nlohmann::json &old_doc,
- nlohmann::json &new_doc, nlohmann::json &changed_doc) {
+ nlohmann::json &new_doc, nlohmann::json &del_doc) {
- for(const auto& it: old_doc.items()) {
+ for(auto it = old_doc.begin(); it != old_doc.end(); ++it) {
new_doc[it.key()] = it.value();
}
- for(const auto& it: document.items()) {
+ for(auto it = document.begin(); it != document.end(); ++it) {
new_doc[it.key()] = it.value();
if(old_doc.count(it.key()) != 0) {
- changed_doc[it.key()] = old_doc[it.key()];
+ // key exists in the stored doc, so it must be reindexed
+ del_doc[it.key()] = old_doc[it.key()];
}
}
}
@@ -302,7 +302,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, const
if(is_update) {
record.operation = UPDATE;
get_document_from_store(get_seq_id_key(seq_id), record.old_doc);
- get_doc_changes(document, record.old_doc, record.new_doc, record.changed_doc);
+ get_doc_changes(document, record.old_doc, record.new_doc, record.del_doc);
}
}
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 75b07790..11e750ea 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -129,6 +129,7 @@ int HttpServer::create_listener() {
ctx.globalconf->server_name = h2o_strdup(nullptr, "", SIZE_MAX);
ctx.globalconf->http2.active_stream_window_size = ACTIVE_STREAM_WINDOW_SIZE;
ctx.globalconf->http2.idle_timeout = REQ_TIMEOUT_MS;
+ ctx.globalconf->max_request_entity_size = (1024 * 1024 * 1024); // 1 GB
ctx.globalconf->http1.req_timeout = REQ_TIMEOUT_MS;
ctx.globalconf->http1.req_io_timeout = REQ_TIMEOUT_MS;
diff --git a/src/index.cpp b/src/index.cpp
index 5721672e..8bf15ec2 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -321,6 +321,43 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,
return Option<>(200);
}
+void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, nlohmann::json& old_doc) {
+ for (auto it = del_doc.begin(); it != del_doc.end(); ++it) {
+ const std::string& field_name = it.key();
+ const auto& search_field_it = search_schema.find(field_name);
+ if(search_field_it == search_schema.end()) {
+ continue;
+ }
+
+ const auto& search_field = search_field_it->second;
+
+ // Go through all the field names and find the keys+values so that they can be removed from in-memory index
+ std::vector<std::string> reindex_tokens;
+ std::vector<std::string> old_tokens;
+ tokenize_doc_field(update_doc, field_name, search_field, reindex_tokens);
+ tokenize_doc_field(old_doc, field_name, search_field, old_tokens);
+
+ if(old_tokens.size() != reindex_tokens.size()) {
+ continue;
+ }
+
+ bool exact_match = true;
+
+ for(size_t i=0; i<reindex_tokens.size(); i++) {
+ const std::string& reindex_val = reindex_tokens[i];
+ const std::string& old_val = old_tokens[i];
+ if(reindex_val != old_val) {
+ exact_match = false;
+ break;
+ }
+ }
+
+ if(exact_match) {
+ del_doc.erase(field_name);
+ }
+ }
+}
+
size_t Index::batch_memory_index(Index *index, std::vector<index_record> & iter_batch,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
@@ -347,19 +384,24 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record> & iter_
}
if(is_update) {
- index->remove(index_rec.seq_id, index_rec.changed_doc);
+ // scrub string fields to reduce delete ops
+ index->scrub_reindex_doc(index_rec.doc, index_rec.del_doc, index_rec.old_doc);
+ index->remove(index_rec.seq_id, index_rec.del_doc);
}
Option<uint32_t> index_mem_op = index->index_in_memory(index_rec.doc, index_rec.seq_id,
default_sorting_field, is_update);
if(!index_mem_op.ok()) {
- index->index_in_memory(index_rec.changed_doc, index_rec.seq_id, default_sorting_field, true);
+ index->index_in_memory(index_rec.del_doc, index_rec.seq_id, default_sorting_field, true);
index_rec.index_failure(index_mem_op.code(), index_mem_op.error());
continue;
}
index_rec.index_success();
- num_indexed++;
+
+ if(!is_update) {
+ num_indexed++;
+ }
}
}
@@ -391,6 +433,7 @@ void Index::insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
art_doc.offsets[i] = kv.second[i];
}
+ //LOG(INFO) << "key: " << key << ", art_doc.id: " << art_doc.id;
art_insert(t, key, key_len, &art_doc, num_hits);
delete [] art_doc.offsets;
art_doc.offsets = nullptr;
@@ -528,6 +571,10 @@ void Index::index_string_field(const std::string & text, const int64_t score, ar
token_to_offsets[token].push_back(i);
}
+ /*if(seq_id == 0) {
+ LOG(INFO) << "field name: " << a_field.name;
+ }*/
+
insert_doc(score, t, seq_id, token_to_offsets);
if(facet_id >= 0) {
@@ -1734,6 +1781,11 @@ void Index::populate_token_positions(const std::vector<art_leaf *>& query_sugges
// a) last element is array_index b) second and third last elements will be largest offset
// (last element is repeated to indicate end of offsets for a given array index)
+ /*uint32_t* offsets = token_leaf->values->offsets.uncompress();
+ for(size_t ii=0; ii < token_leaf->values->offsets.getLength(); ii++) {
+ LOG(INFO) << "offset: " << offsets[ii];
+ }*/
+
uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);
uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?
token_leaf->values->offsets.getLength() :
@@ -1789,8 +1841,8 @@ inline std::vector<art_leaf *> Index::next_suggestion(const std::vector<token_ca
return query_suggestion;
}
-void Index::remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,
- const uint32_t indices_length) {
+void Index::remove_and_shift_offset_index(sorted_array& offset_index, const uint32_t* indices_sorted,
+ const uint32_t indices_length) {
uint32_t *curr_array = offset_index.uncompress();
uint32_t *new_array = new uint32_t[offset_index.getLength()];
@@ -1826,85 +1878,24 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
std::unordered_map<std::string, size_t> facet_to_index;
get_facet_to_index(facet_to_index);
- for(const auto& it: document.items()) {
+ for(auto it = document.begin(); it != document.end(); ++it) {
const std::string& field_name = it.key();
- const auto& search_field = search_schema.find(field_name);
- if(search_field == search_schema.end()) {
+ const auto& search_field_it = search_schema.find(field_name);
+ if(search_field_it == search_schema.end()) {
continue;
}
+ const auto& search_field = search_field_it->second;
+
// Go through all the field names and find the keys+values so that they can be removed from in-memory index
std::vector<std::string> tokens;
- if(search_field->second.type == field_types::STRING) {
- StringUtils::split(document[search_field->first], tokens, " ");
- } else if(search_field->second.type == field_types::STRING_ARRAY) {
- std::vector<std::string> values = document[search_field->first].get<std::vector<std::string>>();
- for(const std::string & value: values) {
- StringUtils::split(value, tokens, " ");
- }
- } else if(search_field->second.type == field_types::INT32) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- int32_t value = document[search_field->first].get<int32_t>();
- encode_int32(value, key);
- tokens.emplace_back((char*)key, KEY_LEN);
- } else if(search_field->second.type == field_types::INT32_ARRAY) {
- std::vector<int32_t> values = document[search_field->first].get<std::vector<int32_t>>();
- for(const int32_t value: values) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- encode_int32(value, key);
- tokens.emplace_back((char*)key, KEY_LEN);
- }
- } else if(search_field->second.type == field_types::INT64) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- int64_t value = document[search_field->first].get<int64_t>();
- encode_int64(value, key);
- tokens.emplace_back((char*)key, KEY_LEN);
- } else if(search_field->second.type == field_types::INT64_ARRAY) {
- std::vector<int64_t> values = document[search_field->first].get<std::vector<int64_t>>();
- for(const int64_t value: values) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- encode_int64(value, key);
- tokens.emplace_back((char*)key, KEY_LEN);
- }
- } else if(search_field->second.type == field_types::FLOAT) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- int64_t value = document[search_field->first].get<int64_t>();
- encode_float(value, key);
- tokens.emplace_back((char*)key, KEY_LEN);
- } else if(search_field->second.type == field_types::FLOAT_ARRAY) {
- std::vector<float> values = document[search_field->first].get<std::vector<float>>();
- for(const float value: values) {
- const int KEY_LEN = 8;
- unsigned char key[KEY_LEN];
- encode_float(value, key);
- tokens.emplace_back((char*)key, KEY_LEN);
- }
- } else if(search_field->second.type == field_types::BOOL) {
- const int KEY_LEN = 1;
- unsigned char key[KEY_LEN];
- bool value = document[search_field->first].get<bool>();
- key[0] = value ? '1' : '0';
- tokens.emplace_back((char*)key, KEY_LEN);
- } else if(search_field->second.type == field_types::BOOL_ARRAY) {
- std::vector<bool> values = document[search_field->first].get<std::vector<bool>>();
- for(const bool value: values) {
- const int KEY_LEN = 1;
- unsigned char key[KEY_LEN];
- key[0] = value ? '1' : '0';
- tokens.emplace_back((char*)key, KEY_LEN);
- }
- }
+ tokenize_doc_field(document, field_name, search_field, tokens);
for(auto & token: tokens) {
const unsigned char *key;
int key_len;
- if(search_field->second.type == field_types::STRING_ARRAY || search_field->second.type == field_types::STRING) {
+ if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {
string_utils.unicode_normalize(token);
key = (const unsigned char *) token.c_str();
key_len = (int) (token.length() + 1);
@@ -1913,7 +1904,7 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
key_len = (int) (token.length());
}
- art_leaf* leaf = (art_leaf *) art_search(search_index.at(search_field->first), key, key_len);
+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name), key, key_len);
if(leaf != nullptr) {
uint32_t seq_id_values[1] = {seq_id};
uint32_t doc_index = leaf->values->ids.indexOf(seq_id);
@@ -1941,7 +1932,7 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
LOG(INFO) << "----";*/
if(leaf->values->ids.getLength() == 0) {
- art_values* values = (art_values*) art_delete(search_index.at(search_field->first), key, key_len);
+ art_values* values = (art_values*) art_delete(search_index.at(field_name), key, key_len);
delete values;
}
}
@@ -1963,6 +1954,74 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
return Option<uint32_t>(seq_id);
}
+void Index::tokenize_doc_field(const nlohmann::json& document, const std::string& field_name, const field& search_field,
+ std::vector<std::string>& tokens) {
+ if(search_field.type == field_types::STRING) {
+ StringUtils::split(document[field_name], tokens, " ");
+ } else if(search_field.type == field_types::STRING_ARRAY) {
+ const std::vector<std::string>& values = document[field_name].get<std::vector<std::string>>();
+ for(const std::string & value: values) {
+ StringUtils::split(value, tokens, " ");
+ }
+ } else if(search_field.type == field_types::INT32) {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+ const int32_t& value = document[field_name].get<int32_t>();
+ encode_int32(value, key);
+ tokens.emplace_back((char*)key, KEY_LEN);
+ } else if(search_field.type == field_types::INT32_ARRAY) {
+ const std::vector<int32_t>& values = document[field_name].get<std::vector<int32_t>>();
+ for(const int32_t value: values) {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+ encode_int32(value, key);
+ tokens.emplace_back((char*)key, KEY_LEN);
+ }
+ } else if(search_field.type == field_types::INT64) {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+ const int64_t& value = document[field_name].get<int64_t>();
+ encode_int64(value, key);
+ tokens.emplace_back((char*)key, KEY_LEN);
+ } else if(search_field.type == field_types::INT64_ARRAY) {
+ const std::vector<int64_t>& values = document[field_name].get<std::vector<int64_t>>();
+ for(const int64_t value: values) {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+ encode_int64(value, key);
+ tokens.emplace_back((char*)key, KEY_LEN);
+ }
+ } else if(search_field.type == field_types::FLOAT) {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+ const int64_t& value = document[field_name].get<int64_t>();
+ encode_float(value, key);
+ tokens.emplace_back((char*)key, KEY_LEN);
+ } else if(search_field.type == field_types::FLOAT_ARRAY) {
+ const std::vector<float>& values = document[field_name].get<std::vector<float>>();
+ for(const float value: values) {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+ encode_float(value, key);
+ tokens.emplace_back((char*)key, KEY_LEN);
+ }
+ } else if(search_field.type == field_types::BOOL) {
+ const int KEY_LEN = 1;
+ unsigned char key[KEY_LEN];
+ const bool& value = document[field_name].get<bool>();
+ key[0] = value ? '1' : '0';
+ tokens.emplace_back((char*)key, KEY_LEN);
+ } else if(search_field.type == field_types::BOOL_ARRAY) {
+ const std::vector<bool>& values = document[field_name].get<std::vector<bool>>();
+ for(const bool value: values) {
+ const int KEY_LEN = 1;
+ unsigned char key[KEY_LEN];
+ key[0] = value ? '1' : '0';
+ tokens.emplace_back((char*)key, KEY_LEN);
+ }
+ }
+}
+
art_leaf* Index::get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len) {
const art_tree *t = search_index.at(field_name);
return (art_leaf*) art_search(t, token, (int) token_len);
diff --git a/src/sorted_array.cpp b/src/sorted_array.cpp
index 60f0a962..9dadc51c 100644
--- a/src/sorted_array.cpp
+++ b/src/sorted_array.cpp
@@ -1,5 +1,6 @@
#include "sorted_array.h"
#include "array_utils.h"
+#include "logger.h"
void sorted_array::load(const uint32_t *sorted_array, const uint32_t array_length) {
min = array_length != 0 ? sorted_array[0] : 0;
@@ -18,28 +19,82 @@ void sorted_array::load(const uint32_t *sorted_array, const uint32_t array_lengt
length_bytes = actual_size;
}
-bool sorted_array::append(uint32_t value) {
+size_t sorted_array::append(uint32_t value) {
uint32_t size_required = sorted_append_size_required(value, length+1);
+ size_t min_expected_size = size_required + FOR_ELE_SIZE;
- if(size_required+FOR_ELE_SIZE > size_bytes) {
- // grow the array first
- size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);
- uint8_t *new_location = (uint8_t *) realloc(in, new_size);
- if(new_location == NULL) {
- abort();
+ /*LOG(INFO) << "value: " << value << ", new_length: " << length+1 << ", size_bytes: " << size_bytes
+ << ", size_required: " << size_required << ", min_expected_size: " << min_expected_size;
+
+ if(value == 0 && size_bytes == 28) {
+ uint32_t* arr = uncompress();
+ for(size_t i=0; i<length; i++) {
+ LOG(INFO) << "ele: " << arr[i];
+ }
+
+ delete [] arr;
+ }*/
+
+ if(value < max) {
+ // we will have to re-encode the whole sequence again
+ uint32_t* arr = uncompress(length+1);
+ size_t i = 0;
+ while(i < length+1) {
+ if(value < arr[i]) {
+ break;
+ }
+ i++;
+ }
+
+ for(size_t j=length; j>i; j--) {
+ arr[j] = arr[j-1];
+ }
+
+ arr[i] = value;
+
+ load(arr, length+1);
+ delete [] arr;
+
+ return i;
+ } else {
+ if(size_bytes < min_expected_size) {
+ // grow the array first
+ size_t new_size = min_expected_size * FOR_GROWTH_FACTOR;
+ uint8_t *new_location = (uint8_t *) realloc(in, new_size);
+ if(new_location == NULL) {
+ abort();
+ }
+ in = new_location;
+ size_bytes = (uint32_t) new_size;
+
+ //LOG(INFO) << "new_size: " << new_size;
}
- in = new_location;
- size_bytes = (uint32_t) new_size;
+
+ uint32_t new_length_bytes = for_append_sorted(in, length, value);
+ if(new_length_bytes == 0) return false;
+
+ length_bytes = new_length_bytes;
+ length++;
+
+ if(value < min) min = value;
+ if(value > max) max = value;
+
+ return length-1;
+ }
+}
+
+bool sorted_array::insert(size_t index, uint32_t value) {
+ if(index >= length) {
+ return false;
}
- uint32_t new_length_bytes = for_append_sorted(in, length, value);
- if(new_length_bytes == 0) return false;
+ uint32_t *curr_array = uncompress(length+1);
+ memmove(&curr_array[index+1], &curr_array[index], sizeof(uint32_t)*(length-index));
+ curr_array[index] = value;
- length_bytes = new_length_bytes;
- length++;
+ load(curr_array, length+1);
- if(value < min) min = value;
- if(value > max) max = value;
+ delete [] curr_array;
return true;
}
@@ -150,20 +205,18 @@ void sorted_array::indexOf(const uint32_t *values, const size_t values_len, uint
binary_search_indices(values, head, tail, low_index, high_index, base, bits, indices);
}
-void sorted_array::remove_values(uint32_t *sorted_values, uint32_t values_length) {
+void sorted_array::remove_values(uint32_t *sorted_values, uint32_t sorted_values_length) {
uint32_t *curr_array = uncompress();
uint32_t *new_array = new uint32_t[length];
uint32_t new_index = 0;
- uint32_t curr_index = 0;
+
uint32_t sorted_values_index = 0;
+ uint32_t curr_index = 0;
while(curr_index < length) {
- if(sorted_values_index < values_length && curr_array[curr_index] >= sorted_values[sorted_values_index]) {
- // skip copying
- if(curr_array[curr_index] == sorted_values[sorted_values_index]) {
- curr_index++;
- }
+ if(sorted_values_index < sorted_values_length && sorted_values[sorted_values_index] == curr_array[curr_index]) {
+ curr_index++;
sorted_values_index++;
} else {
new_array[new_index++] = curr_array[curr_index++];
diff --git a/test/array_test.cpp b/test/array_test.cpp
index 1efa0bd2..d59a7825 100644
--- a/test/array_test.cpp
+++ b/test/array_test.cpp
@@ -45,6 +45,31 @@ TEST(ArrayTest, Append) {
}
}
+TEST(ArrayTest, InsertValues) {
+ std::vector<uint32_t> eles = {10, 1, 4, 5, 7};
+ array arr;
+
+ for(size_t i=0; i < eles.size(); i++) {
+ arr.append(eles[i]);
+ }
+
+ uint32_t insert_arr[2] = {2, 3};
+ arr.insert(2, insert_arr, 2);
+ eles = {10, 1, 2, 3, 4, 5, 7};
+
+ for(size_t i=0; i < eles.size(); i++) {
+ ASSERT_EQ(eles[i], arr.at(i));
+ }
+
+ uint32_t insert_arr2[2] = {20, 25};
+ arr.insert(6, insert_arr2, 2);
+
+ eles = {10, 1, 2, 3, 4, 5, 20, 25, 7};
+ for(size_t i=0; i < eles.size(); i++) {
+ ASSERT_EQ(eles[i], arr.at(i));
+ }
+}
+
TEST(ArrayTest, Uncompress) {
const size_t SIZE = 10*1000;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 7a7537a2..d8d618f8 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1316,7 +1316,7 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 1, fields, "points").get();
}
// try importing records
@@ -1326,8 +1326,8 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {
ASSERT_EQ(18, import_response["num_imported"].get<int>());
// import some new records along with updates
- std::vector<std::string> more_records = {R"({"id": "0", "title": "Wake up, Harry"})",
- R"({"id": "2", "cast": ["Kim Werrel", "Random Wake"]})",
+ std::vector<std::string> more_records = {R"({"id": "0", "title": "The Fifth Harry"})",
+ R"({"id": "2", "cast": ["Chris Fisher", "Rand Alan"]})",
R"({"id": "18", "title": "Back Again Forest", "points": 45, "starring": "Ronald Wells", "cast": ["Dant Saren"]})",
R"({"id": "6", "points": 77})"};
@@ -1344,7 +1344,18 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {
ASSERT_EQ(1, import_results[i].size());
}
- auto results = coll_mul_fields->search("burgundy", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ auto results = coll_mul_fields->search("*", query_fields, "", {}, sort_fields, 0, 30, 1, FREQUENCY, false).get();
+ ASSERT_EQ(19, results["hits"].size());
+
+ ASSERT_EQ(19, coll_mul_fields->get_num_documents());
+
+ results = coll_mul_fields->search("fifth", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(2, results["hits"].size());
+
+ ASSERT_STREQ("The <mark>Fifth</mark> Harry", results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+ ASSERT_STREQ("The Woman in the <mark>Fifth</mark> from Kristin", results["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());
+
+ results = coll_mul_fields->search("burgundy", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(0, results["hits"].size());
results = coll_mul_fields->search("harry", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
@@ -1385,6 +1396,18 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {
ASSERT_FALSE(import_results[1]["success"].get<bool>());
ASSERT_STREQ("A document with id 1 already exists.", import_results[0]["error"].get<std::string>().c_str());
ASSERT_STREQ("A document with id 5 already exists.", import_results[1]["error"].get<std::string>().c_str());
+
+ // update document with verbatim fields, except for points
+ more_records = {R"({"id": "3", "cast":["Matt Damon","Ben Affleck","Minnie Driver"],
+ "points":70,"starring":"Robin Williams","starring_facet":"Robin Williams",
+ "title":"Good Will Hunting"})"};
+
+ import_response = coll_mul_fields->add_many(more_records, true);
+ ASSERT_TRUE(import_response["success"].get<bool>());
+ ASSERT_EQ(1, import_response["num_imported"].get<int>());
+
+ results = coll_mul_fields->search("Good Will Hunting", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(70, results["hits"][0]["document"]["points"].get<uint32_t>());
}
TEST_F(CollectionTest, ImportDocuments) {
@@ -2356,6 +2379,8 @@ TEST_F(CollectionTest, UpdateDocument) {
add_op = coll1->add(doc.dump(), true);
ASSERT_TRUE(add_op.ok());
+ ASSERT_EQ(1, coll1->get_num_documents());
+
res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,
token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();
diff --git a/test/sorted_array_test.cpp b/test/sorted_array_test.cpp
index dfdb9dcc..93351649 100644
--- a/test/sorted_array_test.cpp
+++ b/test/sorted_array_test.cpp
@@ -33,6 +33,72 @@ TEST(SortedArrayTest, Append) {
EXPECT_EQ(arr_small.at(0), 100);
}
+TEST(SortedArrayTest, AppendOutOfOrder) {
+ sorted_array arr;
+ for(size_t i=5; i<=10; i++) {
+ arr.append(i);
+ }
+
+ EXPECT_EQ(6, arr.getLength());
+
+ arr.append(1);
+ arr.append(3);
+ arr.append(2);
+ arr.append(4);
+ arr.append(11);
+ arr.append(14);
+ arr.append(12);
+
+ EXPECT_EQ(13, arr.getLength());
+}
+
+TEST(SortedArrayTest, InsertAtIndex) {
+ std::vector<uint32_t> eles;
+ sorted_array arr;
+ for(size_t i=5; i<=9; i++) {
+ arr.append(i);
+ }
+
+ arr.append(11);
+ eles = {5, 6, 7, 8, 9, 11};
+
+ for(size_t i=0; i < eles.size(); i++) {
+ ASSERT_EQ(eles[i], arr.at(i));
+ }
+
+ arr.insert(0, 1);
+ eles = { 1, 5, 6, 7, 8, 9, 11 };
+
+ for(size_t i=0; i < eles.size(); i++) {
+ ASSERT_EQ(eles[i], arr.at(i));
+ }
+
+ ASSERT_EQ(1, arr.at(0));
+ ASSERT_EQ(5, arr.at(1));
+
+ arr.insert(1, 2);
+ eles = {1, 2, 5, 6, 7, 8, 9, 11};
+ ASSERT_EQ(1, arr.at(0));
+ ASSERT_EQ(2, arr.at(1));
+ ASSERT_EQ(8, arr.getLength());
+
+ for(size_t i=0; i < eles.size(); i++) {
+ ASSERT_EQ(eles[i], arr.at(i));
+ }
+
+ arr.insert(7, 10);
+ eles = { 1, 2, 5, 6, 7, 8, 9, 10, 11};
+ ASSERT_EQ(10, arr.at(7));
+ ASSERT_EQ(11, arr.at(8));
+ ASSERT_EQ(9, arr.getLength());
+
+ for(size_t i=0; i < eles.size(); i++) {
+ ASSERT_EQ(eles[i], arr.at(i));
+ }
+
+ ASSERT_FALSE(arr.insert(9, 12)); // index out of range
+}
+
TEST(SortedArrayTest, Load) {
sorted_array arr;