Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Parallelize parts of indexing.

Kishore Nallanc0602da4fc86
diff --git a/include/index.h b/include/index.h
index 8d1fc1e1..6af2c749 100644
--- a/include/index.h
+++ b/include/index.h
@@ -318,6 +318,11 @@ enum class DIRTY_VALUES {
COERCE_OR_DROP = 4,
};
+struct offsets_facet_hashes_t {
+ std::unordered_map<std::string, std::vector<uint32_t>> offsets;
+ std::vector<uint64_t> facet_hashes;
+};
+
struct index_record {
size_t position; // position of record in the original request
uint32_t seq_id;
@@ -330,9 +335,12 @@ struct index_record {
index_operation_t operation;
bool is_update;
+ // pre-processed data primed for indexing
+ std::unordered_map<std::string, offsets_facet_hashes_t> field_index;
+
Option<bool> indexed; // indicates if the indexing operation was a success
- const DIRTY_VALUES dirty_values;
+ DIRTY_VALUES dirty_values;
index_record(size_t record_pos, uint32_t seq_id, const nlohmann::json& doc, index_operation_t operation,
const DIRTY_VALUES& dirty_values):
@@ -341,6 +349,10 @@ struct index_record {
}
+ index_record(index_record&& rhs) = default;
+
+ index_record& operator=(index_record&& mE) = default;
+
void index_failure(const uint32_t err_code, const std::string & err_msg) {
indexed = Option<bool>(err_code, err_msg);
}
@@ -356,7 +368,7 @@ private:
ThreadPool* thread_pool;
- const uint64_t FACET_ARRAY_DELIMETER = std::numeric_limits<uint64_t>::max();
+ static constexpr const uint64_t FACET_ARRAY_DELIMETER = std::numeric_limits<uint64_t>::max();
std::string name;
@@ -482,11 +494,23 @@ private:
void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;
- void index_string_field(const std::string & text, const int64_t score, art_tree *t, uint32_t seq_id,
- bool is_facet, const field & a_field);
+ static void tokenize_string_with_facets(const std::string& text, bool is_facet, const field& a_field,
+ const std::vector<char>& symbols_to_index,
+ const std::vector<char>& token_separators,
+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
+ std::vector<uint64_t>& facet_hashes);
- void index_string_array_field(const std::vector<std::string> & strings, const int64_t score, art_tree *t,
- uint32_t seq_id, bool is_facet, const field & a_field);
+ void index_strings_field(const int64_t score, art_tree *t,
+ uint32_t seq_id, bool is_facet, const field & a_field,
+ const std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
+ const std::vector<uint64_t>& facet_hashes);
+
+ static void tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,
+ const field& a_field,
+ const std::vector<char>& symbols_to_index,
+ const std::vector<char>& token_separators,
+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
+ std::vector<uint64_t>& facet_hashes);
void collate_included_ids(const std::vector<std::string>& q_included_tokens,
const std::string & field, const uint8_t field_id,
@@ -587,7 +611,14 @@ public:
uint64_t get_distinct_id(const std::vector<std::string>& group_by_fields, const uint32_t seq_id) const;
- void scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, const nlohmann::json& old_doc);
+ static void compute_token_offsets_facets(index_record& record,
+ const std::unordered_map<std::string, field>& search_schema,
+ const std::map<std::string, field>& facet_schema,
+ const std::vector<char>& local_token_separators,
+ const std::vector<char>& local_symbols_to_index);
+
+ static void scrub_reindex_doc(const std::unordered_map<std::string, field>& search_schema,
+ nlohmann::json& update_doc, nlohmann::json& del_doc, const nlohmann::json& old_doc);
static void tokenize_string_field(const nlohmann::json& document,
const field& search_field, std::vector<std::string>& tokens,
@@ -623,16 +654,27 @@ public:
Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);
- Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id,
- const std::string & default_sorting_field,
+ Option<uint32_t> index_in_memory(const index_record& record, uint32_t seq_id,
+ const std::string& default_sorting_field,
const bool is_update);
+ static void validate_and_preprocess(Index *index, std::vector<index_record>& iter_batch,
+ const size_t batch_start_index, const size_t batch_size,
+ const std::string & default_sorting_field,
+ const std::unordered_map<std::string, field> & search_schema,
+ const std::map<std::string, field> & facet_schema,
+ const std::string& fallback_field_type,
+ const std::vector<char>& token_separators,
+ const std::vector<char>& symbols_to_index);
+
static size_t batch_memory_index(Index *index,
std::vector<index_record> & iter_batch,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
const std::map<std::string, field> & facet_schema,
- const std::string& fallback_field_type);
+ const std::string& fallback_field_type,
+ const std::vector<char>& token_separators,
+ const std::vector<char>& symbols_to_index);
//static bool is_point_in_polygon(const Geofence& poly, const GeoCoord& point);
diff --git a/include/option.h b/include/option.h
index 1889a09b..d282712e 100644
--- a/include/option.h
+++ b/include/option.h
@@ -45,4 +45,4 @@ public:
uint32_t code() const {
return error_code;
}
-};
\ No newline at end of file
+};
diff --git a/src/collection.cpp b/src/collection.cpp
index 697b2c97..c92da6ff 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -200,6 +200,9 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
size_t num_indexed = 0;
//bool exceeds_memory_limit = false;
+ // ensures that document IDs are not repeated within the same batch
+ std::set<std::string> batch_doc_ids;
+
for(size_t i=0; i < json_lines.size(); i++) {
const std::string & json_line = json_lines[i];
Option<doc_seq_id_t> doc_seq_id_op = to_doc(json_line, document, operation, dirty_values, id);
@@ -217,6 +220,13 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
record.is_update = !doc_seq_id_op.get().is_new;
if(record.is_update) {
get_document_from_store(get_seq_id_key(seq_id), record.old_doc);
+ } else {
+ const std::string& doc_id = record.doc["id"].get<std::string>();
+ if(batch_doc_ids.find(doc_id) != batch_doc_ids.end()) {
+ record.index_failure(400, "Document with `id` " + doc_id + " already exists in the batch.");
+ } else {
+ batch_doc_ids.emplace(doc_id);
+ }
}
// if `fallback_field_type` or `dynamic_fields` is enabled, update schema first before indexing
@@ -244,7 +254,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
}
*/
- index_records.emplace_back(record);
+ index_records.emplace_back(std::move(record));
if((i+1) % index_batch_size == 0 || i == json_lines.size()-1) {
batch_index(index_records, json_lines, num_indexed);
@@ -255,6 +265,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
document = rec.is_update ? rec.new_doc : rec.doc;
}
index_records.clear();
+ batch_doc_ids.clear();
}
}
@@ -345,7 +356,9 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_t
return validation_op;
}
- index->index_in_memory(document, seq_id, default_sorting_field, op);
+ index_record rec(0, seq_id, document, op, dirty_values);
+ Index::compute_token_offsets_facets(rec, search_schema, facet_schema, token_separators, symbols_to_index);
+ index->index_in_memory(rec, seq_id, default_sorting_field, op);
num_documents += 1;
return Option<>(200);
@@ -354,7 +367,8 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_t
size_t Collection::batch_index_in_memory(std::vector<index_record>& index_records) {
std::unique_lock lock(mutex);
size_t num_indexed = Index::batch_memory_index(index, index_records, default_sorting_field,
- search_schema, facet_schema, fallback_field_type);
+ search_schema, facet_schema, fallback_field_type,
+ token_separators, symbols_to_index);
num_documents += num_indexed;
return num_indexed;
}
diff --git a/src/index.cpp b/src/index.cpp
index f7e8f837..1c61730f 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -151,43 +151,26 @@ int64_t Index::float_to_in64_t(float f) {
return i;
}
-Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t seq_id,
- const std::string & default_sorting_field,
- const bool is_update) {
-
- std::unique_lock lock(mutex);
-
- int64_t points = 0;
-
- if(document.count(default_sorting_field) == 0) {
- if(sort_index.count(default_sorting_field) != 0 && sort_index[default_sorting_field]->count(seq_id)) {
- points = sort_index[default_sorting_field]->at(seq_id);
- } else {
- points = INT64_MIN;
- }
- } else {
- points = get_points_from_doc(document, default_sorting_field);
- }
+void Index::compute_token_offsets_facets(index_record& record,
+ const std::unordered_map<std::string, field>& search_schema,
+ const std::map<std::string, field>& facet_schema,
+ const std::vector<char>& local_token_separators,
+ const std::vector<char>& local_symbols_to_index) {
- if(!is_update) {
- // for updates, the seq_id will already exist
- seq_ids.append(seq_id);
- }
+ const auto& document = record.doc;
- // assumes that validation has already been done
for(const auto& field_pair: search_schema) {
- const std::string & field_name = field_pair.first;
-
+ const std::string& field_name = field_pair.first;
if(document.count(field_name) == 0 || !field_pair.second.index) {
continue;
}
+ offsets_facet_hashes_t offset_facet_hashes;
+
bool is_facet = (facet_schema.count(field_name) != 0);
// non-string, non-geo faceted field should be indexed as faceted string field as well
if(field_pair.second.facet && !field_pair.second.is_string() && !field_pair.second.is_geopoint()) {
- art_tree *t = search_index.at(field_pair.second.faceted_name());
-
if(field_pair.second.is_array()) {
std::vector<std::string> strings;
@@ -208,7 +191,10 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
strings.push_back(std::to_string(value));
}
}
- index_string_array_field(strings, points, t, seq_id, is_facet, field_pair.second);
+
+ tokenize_string_array_with_facets(strings, is_facet, field_pair.second,
+ local_symbols_to_index, local_token_separators,
+ offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);
} else {
std::string text;
@@ -222,17 +208,101 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
text = std::to_string(document[field_name].get<bool>());
}
- index_string_field(text, points, t, seq_id, is_facet, field_pair.second);
+ tokenize_string_with_facets(text, is_facet, field_pair.second,
+ local_symbols_to_index, local_token_separators,
+ offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);
}
}
- if(field_pair.second.type == field_types::STRING) {
- art_tree *t = search_index.at(field_name);
- const std::string & text = document[field_name];
- index_string_field(text, points, t, seq_id, is_facet, field_pair.second);
+ if(field_pair.second.is_string()) {
+
+ if(field_pair.second.type == field_types::STRING) {
+ tokenize_string_with_facets(document[field_name], is_facet, field_pair.second,
+ local_symbols_to_index, local_token_separators,
+ offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);
+ } else {
+ tokenize_string_array_with_facets(document[field_name], is_facet, field_pair.second,
+ local_symbols_to_index, local_token_separators,
+ offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);
+ }
+ }
+
+ if(!offset_facet_hashes.offsets.empty() || !offset_facet_hashes.facet_hashes.empty()) {
+ record.field_index.emplace(field_name, std::move(offset_facet_hashes));
+ }
+ }
+}
+
+Option<uint32_t> Index::index_in_memory(const index_record& record, uint32_t seq_id,
+ const std::string & default_sorting_field,
+ const bool is_update) {
+ std::unique_lock lock(mutex);
+
+ int64_t points = 0;
+ auto& document = record.doc;
+
+ if(document.count(default_sorting_field) == 0) {
+ auto default_sorting_field_it = sort_index.find(default_sorting_field);
+ if(default_sorting_field_it != sort_index.end()) {
+ auto seq_id_it = default_sorting_field_it->second->find(seq_id);
+ if(seq_id_it != default_sorting_field_it->second->end()) {
+ points = seq_id_it->second;
+ } else {
+ points = INT64_MIN;
+ }
+ } else {
+ points = INT64_MIN;
+ }
+ } else {
+ points = get_points_from_doc(document, default_sorting_field);
+ }
+
+ if(!is_update) {
+ // for updates, the seq_id will already exist
+ seq_ids.append(seq_id);
+ }
+
+ // assumes that validation has already been done
+ for(const auto& field_pair: search_schema) {
+ const std::string & field_name = field_pair.first;
+
+ if(document.count(field_name) == 0 || !field_pair.second.index) {
+ continue;
+ }
+
+ bool is_facet = (facet_schema.count(field_name) != 0);
+
+ // non-string, non-geo faceted field should be indexed as faceted string field as well
+ if(field_pair.second.facet && !field_pair.second.is_string() && !field_pair.second.is_geopoint()) {
+ auto field_index_it = record.field_index.find(field_name);
+ if(field_index_it == record.field_index.end()) {
+ continue;
+ }
+
+ art_tree *t = search_index.at(field_pair.second.faceted_name());
+
+ if(field_pair.second.is_array()) {
+ index_strings_field(points, t, seq_id, is_facet, field_pair.second,
+ field_index_it->second.offsets,
+ field_index_it->second.facet_hashes);
+ } else {
+ index_strings_field(points, t, seq_id, is_facet, field_pair.second,
+ field_index_it->second.offsets,
+ field_index_it->second.facet_hashes);
+ }
}
- else if(field_pair.second.type == field_types::INT32) {
+ if(field_pair.second.is_string()) {
+ auto field_index_it = record.field_index.find(field_name);
+ if(field_index_it == record.field_index.end()) {
+ continue;
+ }
+
+ art_tree *t = search_index.at(field_name);
+ index_strings_field(points, t, seq_id, is_facet, field_pair.second,
+ field_index_it->second.offsets,
+ field_index_it->second.facet_hashes);
+ } else if(field_pair.second.type == field_types::INT32) {
auto num_tree = numerical_index.at(field_name);
int32_t value = document[field_name].get<int32_t>();
num_tree->insert(value, seq_id);
@@ -261,9 +331,6 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
for(const auto& term: indexer.GetIndexTerms(point, "")) {
(*geo_index)[term].push_back(seq_id);
}
- } else if(field_pair.second.type == field_types::STRING_ARRAY) {
- art_tree *t = search_index.at(field_name);
- index_string_array_field(document[field_name], points, t, seq_id, is_facet, field_pair.second);
} else if(field_pair.second.type == field_types::GEOPOINT_ARRAY) {
const std::vector<std::vector<double>>& latlongs = document[field_name];
auto geo_index = geopoint_index.at(field_name);
@@ -294,6 +361,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
}
else if(field_pair.second.is_array()) {
+ // all other numerical arrays
auto num_tree = numerical_index.at(field_name);
for(size_t arr_i = 0; arr_i < document[field_name].size(); arr_i++) {
@@ -504,73 +572,126 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3
return Option<>(200);
}
-size_t Index::batch_memory_index(Index *index, std::vector<index_record> & iter_batch,
+void Index::validate_and_preprocess(Index *index, std::vector<index_record>& iter_batch,
+ const size_t batch_start_index, const size_t batch_size,
+ const std::string& default_sorting_field,
+ const std::unordered_map<std::string, field>& search_schema,
+ const std::map<std::string, field>& facet_schema,
+ const std::string& fallback_field_type,
+ const std::vector<char>& token_separators,
+ const std::vector<char>& symbols_to_index) {
+
+ // runs in a partitioned thread
+
+ for(size_t i = 0; i < batch_size; i++) {
+ index_record& index_rec = iter_batch[batch_start_index + i];
+
+ if(!index_rec.indexed.ok()) {
+ // some records could have been invalidated upstream
+ continue;
+ }
+
+ if(index_rec.operation == DELETE) {
+ continue;
+ }
+
+ Option<uint32_t> validation_op = validate_index_in_memory(index_rec.doc, index_rec.seq_id,
+ default_sorting_field,
+ search_schema, facet_schema,
+ index_rec.operation,
+ fallback_field_type,
+ index_rec.dirty_values);
+
+ if(!validation_op.ok()) {
+ index_rec.index_failure(validation_op.code(), validation_op.error());
+ continue;
+ }
+
+ if(index_rec.is_update) {
+ // scrub string fields to reduce delete ops
+ get_doc_changes(index_rec.operation, index_rec.doc, index_rec.old_doc, index_rec.new_doc,
+ index_rec.del_doc);
+ scrub_reindex_doc(search_schema, index_rec.doc, index_rec.del_doc, index_rec.old_doc);
+ }
+
+ compute_token_offsets_facets(index_rec, search_schema, facet_schema, token_separators, symbols_to_index);
+ }
+}
+
+size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_batch,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
const std::map<std::string, field> & facet_schema,
- const std::string& fallback_field_type) {
+ const std::string& fallback_field_type,
+ const std::vector<char>& token_separators,
+ const std::vector<char>& symbols_to_index) {
+
+ const size_t concurrency = 4;
+ const size_t num_threads = std::min(concurrency, iter_batch.size());
+ const size_t window_size = (num_threads == 0) ? 0 :
+ (iter_batch.size() + num_threads - 1) / num_threads; // rounds up
size_t num_indexed = 0;
+ size_t num_processed = 0;
+ std::mutex m_process;
+ std::condition_variable cv_process;
- // ensures that document IDs are not repeated within the same batch
- std::set<std::string> batch_doc_ids;
+ size_t num_queued = 0;
+ size_t batch_index = 0;
- for(auto & index_rec: iter_batch) {
- if(!index_rec.indexed.ok()) {
- // some records could have been invalidated upstream
- continue;
+ for(size_t thread_id = 0; thread_id < num_threads && batch_index < iter_batch.size(); thread_id++) {
+ size_t batch_len = window_size;
+
+ if(batch_index + window_size > iter_batch.size()) {
+ batch_len = iter_batch.size() - batch_index;
}
- if(index_rec.operation != DELETE) {
- const std::string& doc_id = index_rec.doc["id"].get<std::string>();
- if(batch_doc_ids.find(doc_id) != batch_doc_ids.end()) {
- index_rec.index_failure(400, "Document with `id` " + doc_id + " already exists in the import batch.");
- continue;
- }
+ num_queued++;
- batch_doc_ids.emplace(doc_id);
+ index->thread_pool->enqueue([&, batch_index, batch_len]() {
+ validate_and_preprocess(index, iter_batch, batch_index, batch_len, default_sorting_field, search_schema,
+ facet_schema, fallback_field_type,
+ token_separators, symbols_to_index);
- Option<uint32_t> validation_op = validate_index_in_memory(index_rec.doc, index_rec.seq_id,
- default_sorting_field,
- search_schema, facet_schema,
- index_rec.operation,
- fallback_field_type,
- index_rec.dirty_values);
+ std::unique_lock<std::mutex> lock(m_process);
+ num_processed++;
+ cv_process.notify_one();
+ });
- if(!validation_op.ok()) {
- index_rec.index_failure(validation_op.code(), validation_op.error());
- continue;
- }
+ batch_index += batch_len;
+ }
- if(index_rec.is_update) {
- // scrub string fields to reduce delete ops
- get_doc_changes(index_rec.operation, index_rec.doc, index_rec.old_doc, index_rec.new_doc,
- index_rec.del_doc);
- index->scrub_reindex_doc(index_rec.doc, index_rec.del_doc, index_rec.old_doc);
- index->remove(index_rec.seq_id, index_rec.del_doc, index_rec.is_update);
- }
+ std::unique_lock<std::mutex> lock_process(m_process);
+ cv_process.wait(lock_process, [&](){ return num_processed == num_queued; });
- Option<uint32_t> index_mem_op(0);
+ Option<uint32_t> index_mem_op(0);
- try {
- index_mem_op = index->index_in_memory(index_rec.doc, index_rec.seq_id, default_sorting_field, index_rec.is_update);
- } catch(const std::exception& e) {
- const std::string& error_msg = std::string("Fatal error during indexing: ") + e.what();
- LOG(ERROR) << error_msg << ", document: " << index_rec.doc;
- index_mem_op = Option<uint32_t>(500, error_msg);
- }
+ for(auto& index_rec: iter_batch) {
+ if(index_rec.operation == DELETE) {
+ continue;
+ }
- if(!index_mem_op.ok()) {
- index->index_in_memory(index_rec.del_doc, index_rec.seq_id, default_sorting_field, index_rec.is_update);
- index_rec.index_failure(index_mem_op.code(), index_mem_op.error());
- continue;
- }
+ if(!index_rec.indexed.ok()) {
+ // some records could have been invalidated upstream
+ continue;
+ }
- index_rec.index_success();
+ if(index_rec.is_update) {
+ index->remove(index_rec.seq_id, index_rec.del_doc, index_rec.is_update);
+ }
- if(!index_rec.is_update) {
- num_indexed++;
- }
+ try {
+ index_mem_op = index->index_in_memory(index_rec, index_rec.seq_id, default_sorting_field, index_rec.is_update);
+ } catch(const std::exception& e) {
+ const std::string& error_msg = std::string("Fatal error during indexing: ") + e.what();
+ LOG(ERROR) << error_msg << ", document: " << index_rec.doc;
+ index_mem_op = Option<uint32_t>(500, error_msg);
+ }
+
+ index_rec.index_success();
+
+ if(!index_rec.is_update) {
+ num_indexed++;
}
}
@@ -612,23 +733,24 @@ uint64_t Index::facet_token_hash(const field & a_field, const std::string &token
return hash;
}
-void Index::index_string_field(const std::string & text, const int64_t score, art_tree *t,
- uint32_t seq_id, bool is_facet, const field & a_field) {
- std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
+void Index::tokenize_string_with_facets(const std::string& text, bool is_facet, const field& a_field,
+ const std::vector<char>& symbols_to_index,
+ const std::vector<char>& token_separators,
+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
+ std::vector<uint64_t>& facet_hashes) {
Tokenizer tokenizer(text, true, !a_field.is_string(), a_field.locale, symbols_to_index, token_separators);
- std::string token, last_token;
+ std::string token;
+ std::string last_token;
size_t token_index = 0;
- std::vector<uint64_t> facet_hashes;
-
while(tokenizer.next(token, token_index)) {
if(token.empty()) {
continue;
}
if(is_facet) {
- uint64_t hash = facet_token_hash(a_field, token);
+ uint64_t hash = Index::facet_token_hash(a_field, token);
facet_hashes.push_back(hash);
}
@@ -636,16 +758,22 @@ void Index::index_string_field(const std::string & text, const int64_t score, ar
last_token = token;
}
- if(token_to_offsets.empty()) {
- return;
+ if(!token_to_offsets.empty()) {
+ // push 0 for the last occurring token (used for exact match ranking)
+ token_to_offsets[last_token].push_back(0);
}
+}
+
+void Index::index_strings_field(const int64_t score, art_tree *t,
+ uint32_t seq_id, bool is_facet, const field & a_field,
+ const std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
+ const std::vector<uint64_t>& facet_hashes) {
- // push 0 for the last occurring token (used for exact match ranking)
- token_to_offsets[last_token].push_back(0);
+ // requires unique lock from caller
- /*if(seq_id == 0) {
- LOG(INFO) << "field name: " << a_field.name;
- }*/
+ if(token_to_offsets.empty()) {
+ return;
+ }
insert_doc(score, t, seq_id, token_to_offsets);
@@ -662,10 +790,12 @@ void Index::index_string_field(const std::string & text, const int64_t score, ar
}
}
-void Index::index_string_array_field(const std::vector<std::string> & strings, const int64_t score, art_tree *t,
- uint32_t seq_id, bool is_facet, const field & a_field) {
- std::unordered_map<std::string, std::vector<uint32_t>> token_positions;
- std::vector<uint64_t> facet_hashes;
+void Index::tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,
+ const field& a_field,
+ const std::vector<char>& symbols_to_index,
+ const std::vector<char>& token_separators,
+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
+ std::vector<uint64_t>& facet_hashes) {
for(size_t array_index = 0; array_index < strings.size(); array_index++) {
const std::string& str = strings[array_index];
@@ -687,7 +817,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c
//LOG(INFO) << "indexing " << token << ", hash:" << hash;
}
- token_positions[token].push_back(token_index + 1);
+ token_to_offsets[token].push_back(token_index + 1);
token_set.insert(token);
last_token = token;
}
@@ -704,33 +834,15 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c
for(auto& the_token: token_set) {
// repeat last element to indicate end of offsets for this array index
- token_positions[the_token].push_back(token_positions[the_token].back());
+ token_to_offsets[the_token].push_back(token_to_offsets[the_token].back());
// iterate and append this array index to all tokens
- token_positions[the_token].push_back(array_index);
+ token_to_offsets[the_token].push_back(array_index);
}
// push 0 for the last occurring token (used for exact match ranking)
- token_positions[last_token].push_back(0);
+ token_to_offsets[last_token].push_back(0);
}
-
- if(token_positions.empty()) {
- return;
- }
-
- if(is_facet) {
- facet_hash_values_t fhashvalues;
- fhashvalues.length = facet_hashes.size();
- fhashvalues.hashes = new uint64_t[facet_hashes.size()];
-
- for(size_t i = 0; i < facet_hashes.size(); i++) {
- fhashvalues.hashes[i] = facet_hashes[i];
- }
-
- facet_index_v3[a_field.name]->emplace(seq_id, std::move(fhashvalues));
- }
-
- insert_doc(score, t, seq_id, token_positions);
}
void Index::compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type) {
@@ -3541,14 +3653,15 @@ void Index::get_doc_changes(const index_operation_t op, const nlohmann::json& up
}
}
-void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, const nlohmann::json& old_doc) {
+void Index::scrub_reindex_doc(const std::unordered_map<std::string, field>& search_schema,
+ nlohmann::json& update_doc,
+ nlohmann::json& del_doc,
+ const nlohmann::json& old_doc) {
std::vector<std::string> del_keys;
for(auto it = del_doc.cbegin(); it != del_doc.cend(); it++) {
const std::string& field_name = it.key();
- std::shared_lock lock(mutex);
-
const auto& search_field_it = search_schema.find(field_name);
if(search_field_it == search_schema.end()) {
continue;
@@ -3556,8 +3669,6 @@ void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_do
const auto search_field = search_field_it->second; // copy, don't use reference!
- lock.unlock();
-
// compare values between old and update docs:
// if they match, we will remove them from both del and update docs
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 2a7a53b3..151bd3cf 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -1424,7 +1424,7 @@ TEST_F(CollectionSpecificTest, ImportDocumentWithRepeatingIDInTheSameBatch) {
ASSERT_TRUE(nlohmann::json::parse(import_records[0])["success"].get<bool>());
ASSERT_FALSE(nlohmann::json::parse(import_records[1])["success"].get<bool>());
- ASSERT_EQ("Document with `id` 0 already exists in the import batch.",
+ ASSERT_EQ("Document with `id` 0 already exists in the batch.",
nlohmann::json::parse(import_records[1])["error"].get<std::string>());
auto results = coll1->search("levis", {"name"},
@@ -1439,6 +1439,24 @@ TEST_F(CollectionSpecificTest, ImportDocumentWithRepeatingIDInTheSameBatch) {
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("Levis", results["hits"][0]["document"]["name"].get<std::string>());
+ // should allow updates though
+ import_records.clear();
+ import_records.push_back(doc1.dump());
+ import_records.push_back(doc2.dump());
+ import_response = coll1->add_many(import_records, document, index_operation_t::UPDATE);
+
+ ASSERT_TRUE(import_response["success"].get<bool>());
+ ASSERT_EQ(2, import_response["num_imported"].get<int>());
+
+ // should allow upserts also
+ import_records.clear();
+ import_records.push_back(doc1.dump());
+ import_records.push_back(doc2.dump());
+ import_response = coll1->add_many(import_records, document, index_operation_t::UPSERT);
+
+ ASSERT_TRUE(import_response["success"].get<bool>());
+ ASSERT_EQ(2, import_response["num_imported"].get<int>());
+
// repeated ID is rejected even if the first ID is not indexed due to some error
import_records.clear();
doc1.erase("name");
@@ -1455,7 +1473,7 @@ TEST_F(CollectionSpecificTest, ImportDocumentWithRepeatingIDInTheSameBatch) {
ASSERT_FALSE(nlohmann::json::parse(import_records[0])["success"].get<bool>());
ASSERT_FALSE(nlohmann::json::parse(import_records[1])["success"].get<bool>());
- ASSERT_EQ("Document with `id` 100 already exists in the import batch.",
+ ASSERT_EQ("Document with `id` 100 already exists in the batch.",
nlohmann::json::parse(import_records[1])["error"].get<std::string>());
collectionManager.drop_collection("coll1");
diff --git a/test/index_test.cpp b/test/index_test.cpp
index 934fe1c4..554cd31b 100644
--- a/test/index_test.cpp
+++ b/test/index_test.cpp
@@ -26,7 +26,7 @@ TEST(IndexTest, ScrubReindexDoc) {
update_doc1 = old_doc;
del_doc1 = old_doc;
- index.scrub_reindex_doc(update_doc1, del_doc1, old_doc);
+ index.scrub_reindex_doc(search_schema, update_doc1, del_doc1, old_doc);
ASSERT_EQ(1, del_doc1.size());
ASSERT_STREQ("1", del_doc1["id"].get<std::string>().c_str());
@@ -39,7 +39,7 @@ TEST(IndexTest, ScrubReindexDoc) {
del_doc2 = update_doc2;
- index.scrub_reindex_doc(update_doc2, del_doc2, old_doc);
+ index.scrub_reindex_doc(search_schema, update_doc2, del_doc2, old_doc);
ASSERT_EQ(2, del_doc2.size());
ASSERT_STREQ("1", del_doc2["id"].get<std::string>().c_str());
std::vector<std::string> cast = del_doc2["cast"].get<std::vector<std::string>>();
@@ -54,7 +54,7 @@ TEST(IndexTest, ScrubReindexDoc) {
update_doc3["foo"] = "Bar";
del_doc3 = update_doc3;
- index.scrub_reindex_doc(update_doc3, del_doc3, old_doc);
+ index.scrub_reindex_doc(search_schema, update_doc3, del_doc3, old_doc);
ASSERT_EQ(3, del_doc3.size());
ASSERT_STREQ("1", del_doc3["id"].get<std::string>().c_str());
ASSERT_STREQ("The Lawyer", del_doc3["title"].get<std::string>().c_str());