Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Parallelize parts of indexing.
Kishore Nallanc0602da4fc86
diff --git a/include/index.h b/include/index.hindex 8d1fc1e1..6af2c749 100644--- a/include/index.h+++ b/include/index.h@@ -318,6 +318,11 @@ enum class DIRTY_VALUES {COERCE_OR_DROP = 4,};+struct offsets_facet_hashes_t {+ std::unordered_map<std::string, std::vector<uint32_t>> offsets;+ std::vector<uint64_t> facet_hashes;+};+struct index_record {size_t position; // position of record in the original requestuint32_t seq_id;@@ -330,9 +335,12 @@ struct index_record {index_operation_t operation;bool is_update;+ // pre-processed data primed for indexing+ std::unordered_map<std::string, offsets_facet_hashes_t> field_index;+Option<bool> indexed; // indicates if the indexing operation was a success- const DIRTY_VALUES dirty_values;+ DIRTY_VALUES dirty_values;index_record(size_t record_pos, uint32_t seq_id, const nlohmann::json& doc, index_operation_t operation,const DIRTY_VALUES& dirty_values):@@ -341,6 +349,10 @@ struct index_record {}+ index_record(index_record&& rhs) = default;++ index_record& operator=(index_record&& mE) = default;+void index_failure(const uint32_t err_code, const std::string & err_msg) {indexed = Option<bool>(err_code, err_msg);}@@ -356,7 +368,7 @@ private:ThreadPool* thread_pool;- const uint64_t FACET_ARRAY_DELIMETER = std::numeric_limits<uint64_t>::max();+ static constexpr const uint64_t FACET_ARRAY_DELIMETER = std::numeric_limits<uint64_t>::max();std::string name;@@ -482,11 +494,23 @@ private:void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;- void index_string_field(const std::string & text, const int64_t score, art_tree *t, uint32_t seq_id,- bool is_facet, const field & a_field);+ static void tokenize_string_with_facets(const std::string& text, bool is_facet, const field& a_field,+ const std::vector<char>& symbols_to_index,+ const std::vector<char>& token_separators,+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,+ std::vector<uint64_t>& facet_hashes);- void index_string_array_field(const std::vector<std::string> & strings, const int64_t score, art_tree *t,- uint32_t seq_id, bool is_facet, const field & a_field);+ void index_strings_field(const int64_t score, art_tree *t,+ uint32_t seq_id, bool is_facet, const field & a_field,+ const std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,+ const std::vector<uint64_t>& facet_hashes);++ static void tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,+ const field& a_field,+ const std::vector<char>& symbols_to_index,+ const std::vector<char>& token_separators,+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,+ std::vector<uint64_t>& facet_hashes);void collate_included_ids(const std::vector<std::string>& q_included_tokens,const std::string & field, const uint8_t field_id,@@ -587,7 +611,14 @@ public:uint64_t get_distinct_id(const std::vector<std::string>& group_by_fields, const uint32_t seq_id) const;- void scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, const nlohmann::json& old_doc);+ static void compute_token_offsets_facets(index_record& record,+ const std::unordered_map<std::string, field>& search_schema,+ const std::map<std::string, field>& facet_schema,+ const std::vector<char>& local_token_separators,+ const std::vector<char>& local_symbols_to_index);++ static void scrub_reindex_doc(const std::unordered_map<std::string, field>& search_schema,+ nlohmann::json& update_doc, nlohmann::json& del_doc, const nlohmann::json& old_doc);static void tokenize_string_field(const nlohmann::json& document,const field& search_field, std::vector<std::string>& tokens,@@ -623,16 +654,27 @@ public:Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);- Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id,- const std::string & default_sorting_field,+ Option<uint32_t> index_in_memory(const index_record& record, uint32_t seq_id,+ const std::string& default_sorting_field,const bool is_update);+ static void validate_and_preprocess(Index *index, std::vector<index_record>& iter_batch,+ const size_t batch_start_index, const size_t batch_size,+ const std::string & default_sorting_field,+ const std::unordered_map<std::string, field> & search_schema,+ const std::map<std::string, field> & facet_schema,+ const std::string& fallback_field_type,+ const std::vector<char>& token_separators,+ const std::vector<char>& symbols_to_index);+static size_t batch_memory_index(Index *index,std::vector<index_record> & iter_batch,const std::string & default_sorting_field,const std::unordered_map<std::string, field> & search_schema,const std::map<std::string, field> & facet_schema,- const std::string& fallback_field_type);+ const std::string& fallback_field_type,+ const std::vector<char>& token_separators,+ const std::vector<char>& symbols_to_index);//static bool is_point_in_polygon(const Geofence& poly, const GeoCoord& point);diff --git a/include/option.h b/include/option.hindex 1889a09b..d282712e 100644--- a/include/option.h+++ b/include/option.h@@ -45,4 +45,4 @@ public:uint32_t code() const {return error_code;}-};\ No newline at end of file+};diff --git a/src/collection.cpp b/src/collection.cppindex 697b2c97..c92da6ff 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -200,6 +200,9 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmasize_t num_indexed = 0;//bool exceeds_memory_limit = false;+ // ensures that document IDs are not repeated within the same batch+ std::set<std::string> batch_doc_ids;+for(size_t i=0; i < json_lines.size(); i++) {const std::string & json_line = json_lines[i];Option<doc_seq_id_t> doc_seq_id_op = to_doc(json_line, document, operation, dirty_values, id);@@ -217,6 +220,13 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmarecord.is_update = !doc_seq_id_op.get().is_new;if(record.is_update) {get_document_from_store(get_seq_id_key(seq_id), record.old_doc);+ } else {+ const std::string& doc_id = record.doc["id"].get<std::string>();+ if(batch_doc_ids.find(doc_id) != batch_doc_ids.end()) {+ record.index_failure(400, "Document with `id` " + doc_id + " already exists in the batch.");+ } else {+ batch_doc_ids.emplace(doc_id);+ }}// if `fallback_field_type` or `dynamic_fields` is enabled, update schema first before indexing@@ -244,7 +254,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma}*/- index_records.emplace_back(record);+ index_records.emplace_back(std::move(record));if((i+1) % index_batch_size == 0 || i == json_lines.size()-1) {batch_index(index_records, json_lines, num_indexed);@@ -255,6 +265,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmadocument = rec.is_update ? rec.new_doc : rec.doc;}index_records.clear();+ batch_doc_ids.clear();}}@@ -345,7 +356,9 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_treturn validation_op;}- index->index_in_memory(document, seq_id, default_sorting_field, op);+ index_record rec(0, seq_id, document, op, dirty_values);+ Index::compute_token_offsets_facets(rec, search_schema, facet_schema, token_separators, symbols_to_index);+ index->index_in_memory(rec, seq_id, default_sorting_field, op);num_documents += 1;return Option<>(200);@@ -354,7 +367,8 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_tsize_t Collection::batch_index_in_memory(std::vector<index_record>& index_records) {std::unique_lock lock(mutex);size_t num_indexed = Index::batch_memory_index(index, index_records, default_sorting_field,- search_schema, facet_schema, fallback_field_type);+ search_schema, facet_schema, fallback_field_type,+ token_separators, symbols_to_index);num_documents += num_indexed;return num_indexed;}diff --git a/src/index.cpp b/src/index.cppindex f7e8f837..1c61730f 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -151,43 +151,26 @@ int64_t Index::float_to_in64_t(float f) {return i;}-Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t seq_id,- const std::string & default_sorting_field,- const bool is_update) {-- std::unique_lock lock(mutex);-- int64_t points = 0;-- if(document.count(default_sorting_field) == 0) {- if(sort_index.count(default_sorting_field) != 0 && sort_index[default_sorting_field]->count(seq_id)) {- points = sort_index[default_sorting_field]->at(seq_id);- } else {- points = INT64_MIN;- }- } else {- points = get_points_from_doc(document, default_sorting_field);- }+void Index::compute_token_offsets_facets(index_record& record,+ const std::unordered_map<std::string, field>& search_schema,+ const std::map<std::string, field>& facet_schema,+ const std::vector<char>& local_token_separators,+ const std::vector<char>& local_symbols_to_index) {- if(!is_update) {- // for updates, the seq_id will already exist- seq_ids.append(seq_id);- }+ const auto& document = record.doc;- // assumes that validation has already been donefor(const auto& field_pair: search_schema) {- const std::string & field_name = field_pair.first;-+ const std::string& field_name = field_pair.first;if(document.count(field_name) == 0 || !field_pair.second.index) {continue;}+ offsets_facet_hashes_t offset_facet_hashes;+bool is_facet = (facet_schema.count(field_name) != 0);// non-string, non-geo faceted field should be indexed as faceted string field as wellif(field_pair.second.facet && !field_pair.second.is_string() && !field_pair.second.is_geopoint()) {- art_tree *t = search_index.at(field_pair.second.faceted_name());-if(field_pair.second.is_array()) {std::vector<std::string> strings;@@ -208,7 +191,10 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tstrings.push_back(std::to_string(value));}}- index_string_array_field(strings, points, t, seq_id, is_facet, field_pair.second);++ tokenize_string_array_with_facets(strings, is_facet, field_pair.second,+ local_symbols_to_index, local_token_separators,+ offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);} else {std::string text;@@ -222,17 +208,101 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_ttext = std::to_string(document[field_name].get<bool>());}- index_string_field(text, points, t, seq_id, is_facet, field_pair.second);+ tokenize_string_with_facets(text, is_facet, field_pair.second,+ local_symbols_to_index, local_token_separators,+ offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);}}- if(field_pair.second.type == field_types::STRING) {- art_tree *t = search_index.at(field_name);- const std::string & text = document[field_name];- index_string_field(text, points, t, seq_id, is_facet, field_pair.second);+ if(field_pair.second.is_string()) {++ if(field_pair.second.type == field_types::STRING) {+ tokenize_string_with_facets(document[field_name], is_facet, field_pair.second,+ local_symbols_to_index, local_token_separators,+ offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);+ } else {+ tokenize_string_array_with_facets(document[field_name], is_facet, field_pair.second,+ local_symbols_to_index, local_token_separators,+ offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);+ }+ }++ if(!offset_facet_hashes.offsets.empty() || !offset_facet_hashes.facet_hashes.empty()) {+ record.field_index.emplace(field_name, std::move(offset_facet_hashes));+ }+ }+}++Option<uint32_t> Index::index_in_memory(const index_record& record, uint32_t seq_id,+ const std::string & default_sorting_field,+ const bool is_update) {+ std::unique_lock lock(mutex);++ int64_t points = 0;+ auto& document = record.doc;++ if(document.count(default_sorting_field) == 0) {+ auto default_sorting_field_it = sort_index.find(default_sorting_field);+ if(default_sorting_field_it != sort_index.end()) {+ auto seq_id_it = default_sorting_field_it->second->find(seq_id);+ if(seq_id_it != default_sorting_field_it->second->end()) {+ points = seq_id_it->second;+ } else {+ points = INT64_MIN;+ }+ } else {+ points = INT64_MIN;+ }+ } else {+ points = get_points_from_doc(document, default_sorting_field);+ }++ if(!is_update) {+ // for updates, the seq_id will already exist+ seq_ids.append(seq_id);+ }++ // assumes that validation has already been done+ for(const auto& field_pair: search_schema) {+ const std::string & field_name = field_pair.first;++ if(document.count(field_name) == 0 || !field_pair.second.index) {+ continue;+ }++ bool is_facet = (facet_schema.count(field_name) != 0);++ // non-string, non-geo faceted field should be indexed as faceted string field as well+ if(field_pair.second.facet && !field_pair.second.is_string() && !field_pair.second.is_geopoint()) {+ auto field_index_it = record.field_index.find(field_name);+ if(field_index_it == record.field_index.end()) {+ continue;+ }++ art_tree *t = search_index.at(field_pair.second.faceted_name());++ if(field_pair.second.is_array()) {+ index_strings_field(points, t, seq_id, is_facet, field_pair.second,+ field_index_it->second.offsets,+ field_index_it->second.facet_hashes);+ } else {+ index_strings_field(points, t, seq_id, is_facet, field_pair.second,+ field_index_it->second.offsets,+ field_index_it->second.facet_hashes);+ }}- else if(field_pair.second.type == field_types::INT32) {+ if(field_pair.second.is_string()) {+ auto field_index_it = record.field_index.find(field_name);+ if(field_index_it == record.field_index.end()) {+ continue;+ }++ art_tree *t = search_index.at(field_name);+ index_strings_field(points, t, seq_id, is_facet, field_pair.second,+ field_index_it->second.offsets,+ field_index_it->second.facet_hashes);+ } else if(field_pair.second.type == field_types::INT32) {auto num_tree = numerical_index.at(field_name);int32_t value = document[field_name].get<int32_t>();num_tree->insert(value, seq_id);@@ -261,9 +331,6 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tfor(const auto& term: indexer.GetIndexTerms(point, "")) {(*geo_index)[term].push_back(seq_id);}- } else if(field_pair.second.type == field_types::STRING_ARRAY) {- art_tree *t = search_index.at(field_name);- index_string_array_field(document[field_name], points, t, seq_id, is_facet, field_pair.second);} else if(field_pair.second.type == field_types::GEOPOINT_ARRAY) {const std::vector<std::vector<double>>& latlongs = document[field_name];auto geo_index = geopoint_index.at(field_name);@@ -294,6 +361,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t}else if(field_pair.second.is_array()) {+ // all other numerical arraysauto num_tree = numerical_index.at(field_name);for(size_t arr_i = 0; arr_i < document[field_name].size(); arr_i++) {@@ -504,73 +572,126 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3return Option<>(200);}-size_t Index::batch_memory_index(Index *index, std::vector<index_record> & iter_batch,+void Index::validate_and_preprocess(Index *index, std::vector<index_record>& iter_batch,+ const size_t batch_start_index, const size_t batch_size,+ const std::string& default_sorting_field,+ const std::unordered_map<std::string, field>& search_schema,+ const std::map<std::string, field>& facet_schema,+ const std::string& fallback_field_type,+ const std::vector<char>& token_separators,+ const std::vector<char>& symbols_to_index) {++ // runs in a partitioned thread++ for(size_t i = 0; i < batch_size; i++) {+ index_record& index_rec = iter_batch[batch_start_index + i];++ if(!index_rec.indexed.ok()) {+ // some records could have been invalidated upstream+ continue;+ }++ if(index_rec.operation == DELETE) {+ continue;+ }++ Option<uint32_t> validation_op = validate_index_in_memory(index_rec.doc, index_rec.seq_id,+ default_sorting_field,+ search_schema, facet_schema,+ index_rec.operation,+ fallback_field_type,+ index_rec.dirty_values);++ if(!validation_op.ok()) {+ index_rec.index_failure(validation_op.code(), validation_op.error());+ continue;+ }++ if(index_rec.is_update) {+ // scrub string fields to reduce delete ops+ get_doc_changes(index_rec.operation, index_rec.doc, index_rec.old_doc, index_rec.new_doc,+ index_rec.del_doc);+ scrub_reindex_doc(search_schema, index_rec.doc, index_rec.del_doc, index_rec.old_doc);+ }++ compute_token_offsets_facets(index_rec, search_schema, facet_schema, token_separators, symbols_to_index);+ }+}++size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_batch,const std::string & default_sorting_field,const std::unordered_map<std::string, field> & search_schema,const std::map<std::string, field> & facet_schema,- const std::string& fallback_field_type) {+ const std::string& fallback_field_type,+ const std::vector<char>& token_separators,+ const std::vector<char>& symbols_to_index) {++ const size_t concurrency = 4;+ const size_t num_threads = std::min(concurrency, iter_batch.size());+ const size_t window_size = (num_threads == 0) ? 0 :+ (iter_batch.size() + num_threads - 1) / num_threads; // rounds upsize_t num_indexed = 0;+ size_t num_processed = 0;+ std::mutex m_process;+ std::condition_variable cv_process;- // ensures that document IDs are not repeated within the same batch- std::set<std::string> batch_doc_ids;+ size_t num_queued = 0;+ size_t batch_index = 0;- for(auto & index_rec: iter_batch) {- if(!index_rec.indexed.ok()) {- // some records could have been invalidated upstream- continue;+ for(size_t thread_id = 0; thread_id < num_threads && batch_index < iter_batch.size(); thread_id++) {+ size_t batch_len = window_size;++ if(batch_index + window_size > iter_batch.size()) {+ batch_len = iter_batch.size() - batch_index;}- if(index_rec.operation != DELETE) {- const std::string& doc_id = index_rec.doc["id"].get<std::string>();- if(batch_doc_ids.find(doc_id) != batch_doc_ids.end()) {- index_rec.index_failure(400, "Document with `id` " + doc_id + " already exists in the import batch.");- continue;- }+ num_queued++;- batch_doc_ids.emplace(doc_id);+ index->thread_pool->enqueue([&, batch_index, batch_len]() {+ validate_and_preprocess(index, iter_batch, batch_index, batch_len, default_sorting_field, search_schema,+ facet_schema, fallback_field_type,+ token_separators, symbols_to_index);- Option<uint32_t> validation_op = validate_index_in_memory(index_rec.doc, index_rec.seq_id,- default_sorting_field,- search_schema, facet_schema,- index_rec.operation,- fallback_field_type,- index_rec.dirty_values);+ std::unique_lock<std::mutex> lock(m_process);+ num_processed++;+ cv_process.notify_one();+ });- if(!validation_op.ok()) {- index_rec.index_failure(validation_op.code(), validation_op.error());- continue;- }+ batch_index += batch_len;+ }- if(index_rec.is_update) {- // scrub string fields to reduce delete ops- get_doc_changes(index_rec.operation, index_rec.doc, index_rec.old_doc, index_rec.new_doc,- index_rec.del_doc);- index->scrub_reindex_doc(index_rec.doc, index_rec.del_doc, index_rec.old_doc);- index->remove(index_rec.seq_id, index_rec.del_doc, index_rec.is_update);- }+ std::unique_lock<std::mutex> lock_process(m_process);+ cv_process.wait(lock_process, [&](){ return num_processed == num_queued; });- Option<uint32_t> index_mem_op(0);+ Option<uint32_t> index_mem_op(0);- try {- index_mem_op = index->index_in_memory(index_rec.doc, index_rec.seq_id, default_sorting_field, index_rec.is_update);- } catch(const std::exception& e) {- const std::string& error_msg = std::string("Fatal error during indexing: ") + e.what();- LOG(ERROR) << error_msg << ", document: " << index_rec.doc;- index_mem_op = Option<uint32_t>(500, error_msg);- }+ for(auto& index_rec: iter_batch) {+ if(index_rec.operation == DELETE) {+ continue;+ }- if(!index_mem_op.ok()) {- index->index_in_memory(index_rec.del_doc, index_rec.seq_id, default_sorting_field, index_rec.is_update);- index_rec.index_failure(index_mem_op.code(), index_mem_op.error());- continue;- }+ if(!index_rec.indexed.ok()) {+ // some records could have been invalidated upstream+ continue;+ }- index_rec.index_success();+ if(index_rec.is_update) {+ index->remove(index_rec.seq_id, index_rec.del_doc, index_rec.is_update);+ }- if(!index_rec.is_update) {- num_indexed++;- }+ try {+ index_mem_op = index->index_in_memory(index_rec, index_rec.seq_id, default_sorting_field, index_rec.is_update);+ } catch(const std::exception& e) {+ const std::string& error_msg = std::string("Fatal error during indexing: ") + e.what();+ LOG(ERROR) << error_msg << ", document: " << index_rec.doc;+ index_mem_op = Option<uint32_t>(500, error_msg);+ }++ index_rec.index_success();++ if(!index_rec.is_update) {+ num_indexed++;}}@@ -612,23 +733,24 @@ uint64_t Index::facet_token_hash(const field & a_field, const std::string &tokenreturn hash;}-void Index::index_string_field(const std::string & text, const int64_t score, art_tree *t,- uint32_t seq_id, bool is_facet, const field & a_field) {- std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;+void Index::tokenize_string_with_facets(const std::string& text, bool is_facet, const field& a_field,+ const std::vector<char>& symbols_to_index,+ const std::vector<char>& token_separators,+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,+ std::vector<uint64_t>& facet_hashes) {Tokenizer tokenizer(text, true, !a_field.is_string(), a_field.locale, symbols_to_index, token_separators);- std::string token, last_token;+ std::string token;+ std::string last_token;size_t token_index = 0;- std::vector<uint64_t> facet_hashes;-while(tokenizer.next(token, token_index)) {if(token.empty()) {continue;}if(is_facet) {- uint64_t hash = facet_token_hash(a_field, token);+ uint64_t hash = Index::facet_token_hash(a_field, token);facet_hashes.push_back(hash);}@@ -636,16 +758,22 @@ void Index::index_string_field(const std::string & text, const int64_t score, arlast_token = token;}- if(token_to_offsets.empty()) {- return;+ if(!token_to_offsets.empty()) {+ // push 0 for the last occurring token (used for exact match ranking)+ token_to_offsets[last_token].push_back(0);}+}++void Index::index_strings_field(const int64_t score, art_tree *t,+ uint32_t seq_id, bool is_facet, const field & a_field,+ const std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,+ const std::vector<uint64_t>& facet_hashes) {- // push 0 for the last occurring token (used for exact match ranking)- token_to_offsets[last_token].push_back(0);+ // requires unique lock from caller- /*if(seq_id == 0) {- LOG(INFO) << "field name: " << a_field.name;- }*/+ if(token_to_offsets.empty()) {+ return;+ }insert_doc(score, t, seq_id, token_to_offsets);@@ -662,10 +790,12 @@ void Index::index_string_field(const std::string & text, const int64_t score, ar}}-void Index::index_string_array_field(const std::vector<std::string> & strings, const int64_t score, art_tree *t,- uint32_t seq_id, bool is_facet, const field & a_field) {- std::unordered_map<std::string, std::vector<uint32_t>> token_positions;- std::vector<uint64_t> facet_hashes;+void Index::tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,+ const field& a_field,+ const std::vector<char>& symbols_to_index,+ const std::vector<char>& token_separators,+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,+ std::vector<uint64_t>& facet_hashes) {for(size_t array_index = 0; array_index < strings.size(); array_index++) {const std::string& str = strings[array_index];@@ -687,7 +817,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c//LOG(INFO) << "indexing " << token << ", hash:" << hash;}- token_positions[token].push_back(token_index + 1);+ token_to_offsets[token].push_back(token_index + 1);token_set.insert(token);last_token = token;}@@ -704,33 +834,15 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, cfor(auto& the_token: token_set) {// repeat last element to indicate end of offsets for this array index- token_positions[the_token].push_back(token_positions[the_token].back());+ token_to_offsets[the_token].push_back(token_to_offsets[the_token].back());// iterate and append this array index to all tokens- token_positions[the_token].push_back(array_index);+ token_to_offsets[the_token].push_back(array_index);}// push 0 for the last occurring token (used for exact match ranking)- token_positions[last_token].push_back(0);+ token_to_offsets[last_token].push_back(0);}-- if(token_positions.empty()) {- return;- }-- if(is_facet) {- facet_hash_values_t fhashvalues;- fhashvalues.length = facet_hashes.size();- fhashvalues.hashes = new uint64_t[facet_hashes.size()];-- for(size_t i = 0; i < facet_hashes.size(); i++) {- fhashvalues.hashes[i] = facet_hashes[i];- }-- facet_index_v3[a_field.name]->emplace(seq_id, std::move(fhashvalues));- }-- insert_doc(score, t, seq_id, token_positions);}void Index::compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type) {@@ -3541,14 +3653,15 @@ void Index::get_doc_changes(const index_operation_t op, const nlohmann::json& up}}-void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, const nlohmann::json& old_doc) {+void Index::scrub_reindex_doc(const std::unordered_map<std::string, field>& search_schema,+ nlohmann::json& update_doc,+ nlohmann::json& del_doc,+ const nlohmann::json& old_doc) {std::vector<std::string> del_keys;for(auto it = del_doc.cbegin(); it != del_doc.cend(); it++) {const std::string& field_name = it.key();- std::shared_lock lock(mutex);-const auto& search_field_it = search_schema.find(field_name);if(search_field_it == search_schema.end()) {continue;@@ -3556,8 +3669,6 @@ void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doconst auto search_field = search_field_it->second; // copy, don't use reference!- lock.unlock();-// compare values between old and update docs:// if they match, we will remove them from both del and update docsdiff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 2a7a53b3..151bd3cf 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -1424,7 +1424,7 @@ TEST_F(CollectionSpecificTest, ImportDocumentWithRepeatingIDInTheSameBatch) {ASSERT_TRUE(nlohmann::json::parse(import_records[0])["success"].get<bool>());ASSERT_FALSE(nlohmann::json::parse(import_records[1])["success"].get<bool>());- ASSERT_EQ("Document with `id` 0 already exists in the import batch.",+ ASSERT_EQ("Document with `id` 0 already exists in the batch.",nlohmann::json::parse(import_records[1])["error"].get<std::string>());auto results = coll1->search("levis", {"name"},@@ -1439,6 +1439,24 @@ TEST_F(CollectionSpecificTest, ImportDocumentWithRepeatingIDInTheSameBatch) {ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("Levis", results["hits"][0]["document"]["name"].get<std::string>());+ // should allow updates though+ import_records.clear();+ import_records.push_back(doc1.dump());+ import_records.push_back(doc2.dump());+ import_response = coll1->add_many(import_records, document, index_operation_t::UPDATE);++ ASSERT_TRUE(import_response["success"].get<bool>());+ ASSERT_EQ(2, import_response["num_imported"].get<int>());++ // should allow upserts also+ import_records.clear();+ import_records.push_back(doc1.dump());+ import_records.push_back(doc2.dump());+ import_response = coll1->add_many(import_records, document, index_operation_t::UPSERT);++ ASSERT_TRUE(import_response["success"].get<bool>());+ ASSERT_EQ(2, import_response["num_imported"].get<int>());+// repeated ID is rejected even if the first ID is not indexed due to some errorimport_records.clear();doc1.erase("name");@@ -1455,7 +1473,7 @@ TEST_F(CollectionSpecificTest, ImportDocumentWithRepeatingIDInTheSameBatch) {ASSERT_FALSE(nlohmann::json::parse(import_records[0])["success"].get<bool>());ASSERT_FALSE(nlohmann::json::parse(import_records[1])["success"].get<bool>());- ASSERT_EQ("Document with `id` 100 already exists in the import batch.",+ ASSERT_EQ("Document with `id` 100 already exists in the batch.",nlohmann::json::parse(import_records[1])["error"].get<std::string>());collectionManager.drop_collection("coll1");diff --git a/test/index_test.cpp b/test/index_test.cppindex 934fe1c4..554cd31b 100644--- a/test/index_test.cpp+++ b/test/index_test.cpp@@ -26,7 +26,7 @@ TEST(IndexTest, ScrubReindexDoc) {update_doc1 = old_doc;del_doc1 = old_doc;- index.scrub_reindex_doc(update_doc1, del_doc1, old_doc);+ index.scrub_reindex_doc(search_schema, update_doc1, del_doc1, old_doc);ASSERT_EQ(1, del_doc1.size());ASSERT_STREQ("1", del_doc1["id"].get<std::string>().c_str());@@ -39,7 +39,7 @@ TEST(IndexTest, ScrubReindexDoc) {del_doc2 = update_doc2;- index.scrub_reindex_doc(update_doc2, del_doc2, old_doc);+ index.scrub_reindex_doc(search_schema, update_doc2, del_doc2, old_doc);ASSERT_EQ(2, del_doc2.size());ASSERT_STREQ("1", del_doc2["id"].get<std::string>().c_str());std::vector<std::string> cast = del_doc2["cast"].get<std::vector<std::string>>();@@ -54,7 +54,7 @@ TEST(IndexTest, ScrubReindexDoc) {update_doc3["foo"] = "Bar";del_doc3 = update_doc3;- index.scrub_reindex_doc(update_doc3, del_doc3, old_doc);+ index.scrub_reindex_doc(search_schema, update_doc3, del_doc3, old_doc);ASSERT_EQ(3, del_doc3.size());ASSERT_STREQ("1", del_doc3["id"].get<std::string>().c_str());ASSERT_STREQ("The Lawyer", del_doc3["title"].get<std::string>().c_str());