Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add guard for really large documents eating memory during load.
Kishore Nallan7f348250b626
diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 742ea395..922fee38 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1222,6 +1222,7 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectionsize_t num_found_docs = 0;size_t num_valid_docs = 0;size_t num_indexed_docs = 0;+ size_t batch_doc_str_size = 0;auto begin = std::chrono::high_resolution_clock::now();@@ -1230,14 +1231,17 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectionconst uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());nlohmann::json document;+ const std::string& doc_string = iter->value().ToString();try {- document = nlohmann::json::parse(iter->value().ToString());+ document = nlohmann::json::parse(doc_string);} catch(const std::exception& e) {LOG(ERROR) << "JSON error: " << e.what();return Option<bool>(400, "Bad JSON.");}+ batch_doc_str_size += doc_string.size();+if(collection->get_enable_nested_fields()) {std::vector<field> flattened_fields;field::flatten_doc(document, collection->get_nested_fields(), true, flattened_fields);@@ -1254,10 +1258,14 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectioniter->Next();bool last_record = !(iter->Valid() && iter->key().starts_with(seq_id_prefix));+ // if expected memory usage exceeds 250M, we index the accumulated set without caring about batch size+ bool exceeds_batch_mem_threshold = ((batch_doc_str_size * 7) > (250 * 1014 * 1024));+// batch must match atleast the number of shards- if((num_valid_docs % batch_size == 0) || last_record) {+ if(exceeds_batch_mem_threshold || (num_valid_docs % batch_size == 0) || last_record) {size_t num_records = index_records.size();size_t num_indexed = collection->batch_index_in_memory(index_records);+ batch_doc_str_size = 0;if(num_indexed != num_records) {const Option<std::string> & index_error_op = get_first_index_error(index_records);