Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle updates of repeating records within the same batch.
Kishore Nallanb14458efd487
diff --git a/src/collection.cpp b/src/collection.cppindex f690e5f3..627454a8 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -223,22 +223,28 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma// NOTE: we overwrite the input json_lines with result to avoid memory pressurerecord.is_update = false;+ bool repeated_doc = false;if(!doc_seq_id_op.ok()) {record.index_failure(doc_seq_id_op.code(), doc_seq_id_op.error());} else {+ const std::string& doc_id = record.doc["id"].get<std::string>();+ repeated_doc = (batch_doc_ids.find(doc_id) != batch_doc_ids.end());++ if(repeated_doc) {+ // when a document repeats, we send the batch until this document so that we can deal with conflicts+ i--;+ goto do_batched_index;+ }+record.is_update = !doc_seq_id_op.get().is_new;+if(record.is_update) {get_document_from_store(get_seq_id_key(seq_id), record.old_doc);- } else {- const std::string& doc_id = record.doc["id"].get<std::string>();- if(batch_doc_ids.find(doc_id) != batch_doc_ids.end()) {- record.index_failure(400, "Document with `id` " + doc_id + " already exists in the batch.");- } else {- batch_doc_ids.emplace(doc_id);- }}+ batch_doc_ids.insert(doc_id);+// if `fallback_field_type` or `dynamic_fields` is enabled, update schema first before indexingif(!fallback_field_type.empty() || !dynamic_fields.empty()) {Option<bool> schema_change_op = check_and_update_schema(record.doc, dirty_values);@@ -248,25 +254,11 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma}}- /*- // check for memory threshold before allowing subsequent batches- if(is_exceeding_memory_threshold()) {- exceeds_memory_limit = true;- }-- if(exceeds_memory_limit) {- nlohmann::json index_res;- index_res["error"] = "Max memory ratio exceeded.";- index_res["success"] = false;- index_res["document"] = json_line;- json_lines[i] = index_res.dump();- record.index_failure(500, "Max memory ratio exceeded.");- }- */-index_records.emplace_back(std::move(record));- if((i+1) % index_batch_size == 0 || i == json_lines.size()-1) {+ do_batched_index:++ if((i+1) % index_batch_size == 0 || i == json_lines.size()-1 || repeated_doc) {batch_index(index_records, json_lines, num_indexed);// to return the document for the single doc add casesdiff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex c148b659..3c11518f 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -1452,7 +1452,7 @@ TEST_F(CollectionSpecificTest, ImportDocumentWithRepeatingIDInTheSameBatch) {ASSERT_TRUE(nlohmann::json::parse(import_records[0])["success"].get<bool>());ASSERT_FALSE(nlohmann::json::parse(import_records[1])["success"].get<bool>());- ASSERT_EQ("Document with `id` 0 already exists in the batch.",+ ASSERT_EQ("A document with id 0 already exists.",nlohmann::json::parse(import_records[1])["error"].get<std::string>());auto results = coll1->search("levis", {"name"},@@ -1485,7 +1485,7 @@ TEST_F(CollectionSpecificTest, ImportDocumentWithRepeatingIDInTheSameBatch) {ASSERT_TRUE(import_response["success"].get<bool>());ASSERT_EQ(2, import_response["num_imported"].get<int>());- // repeated ID is rejected even if the first ID is not indexed due to some error+ // repeated ID is NOT rejected if the first ID is not indexed due to some errorimport_records.clear();doc1.erase("name");doc1["id"] = "100";@@ -1496,17 +1496,187 @@ TEST_F(CollectionSpecificTest, ImportDocumentWithRepeatingIDInTheSameBatch) {import_response = coll1->add_many(import_records, document);+ LOG(INFO) << import_response;+ASSERT_FALSE(import_response["success"].get<bool>());- ASSERT_EQ(0, import_response["num_imported"].get<int>());+ ASSERT_EQ(1, import_response["num_imported"].get<int>());ASSERT_FALSE(nlohmann::json::parse(import_records[0])["success"].get<bool>());- ASSERT_FALSE(nlohmann::json::parse(import_records[1])["success"].get<bool>());- ASSERT_EQ("Document with `id` 100 already exists in the batch.",- nlohmann::json::parse(import_records[1])["error"].get<std::string>());+ ASSERT_EQ("Field `name` has been declared in the schema, but is not found in the document.",+ nlohmann::json::parse(import_records[0])["error"].get<std::string>());++ ASSERT_TRUE(nlohmann::json::parse(import_records[1])["success"].get<bool>());collectionManager.drop_collection("coll1");}++TEST_F(CollectionSpecificTest, UpdateOfTwoDocsWithSameIdWithinSameBatch) {+ std::vector<field> fields = {field("last_chance", field_types::BOOL, false, true),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["points"] = 100;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());++ // second update should reflect the result of first update+ std::vector<std::string> updates = {+ R"({"id": "0", "last_chance": false})",+ R"({"id": "0", "points": 200})",+ };++ nlohmann::json update_doc;+ auto import_response = coll1->add_many(updates, update_doc, UPDATE);+ ASSERT_TRUE(import_response["success"].get<bool>());+ ASSERT_EQ(2, import_response["num_imported"].get<int>());++ auto results = coll1->search("*", {},+ "", {}, {}, {0}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true).get();++ LOG(INFO) << results;++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_TRUE(results["hits"][0]["document"].contains("last_chance"));+ ASSERT_EQ(false, results["hits"][0]["document"]["last_chance"].get<bool>());++ ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().at("points")->size());+ ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().at("last_chance")->size());+}++TEST_F(CollectionSpecificTest, UpsertOfTwoDocsWithSameIdWithinSameBatch) {+ std::vector<field> fields = {field("last_chance", field_types::BOOL, false, true),+ field("points", field_types::INT32, false, true),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["points"] = 100;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());++ // first upsert removes both fields, so second upsert should only insert "points"+ std::vector<std::string> upserts = {+ R"({"id": "0", "last_chance": true})",+ R"({"id": "0", "points": 200})",+ };++ nlohmann::json update_doc;+ auto import_response = coll1->add_many(upserts, update_doc, UPSERT);+ ASSERT_TRUE(import_response["success"].get<bool>());+ ASSERT_EQ(2, import_response["num_imported"].get<int>());++ auto results = coll1->search("*", {},+ "", {}, {}, {0}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_TRUE(results["hits"][0]["document"].contains("points"));+ ASSERT_FALSE(results["hits"][0]["document"].contains("last_chance"));+ ASSERT_EQ(200, results["hits"][0]["document"]["points"].get<int32_t>());++ ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().at("points")->size());+ ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().at("last_chance")->size());++ // update without doc id++ upserts = {+ R"({"last_chance": true})",+ };++ import_response = coll1->add_many(upserts, update_doc, UPDATE);+ ASSERT_FALSE(import_response["success"].get<bool>());+ ASSERT_EQ(0, import_response["num_imported"].get<int>());+}++TEST_F(CollectionSpecificTest, UpdateUpsertOfDocWithMissingFields) {+ std::vector<field> fields = {field("last_chance", field_types::BOOL, false, true),+ field("points", field_types::INT32, false, true),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["last_chance"] = true;+ doc1["points"] = 100;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());++ // upsert doc with missing fields: should be removed from index+ std::vector<std::string> upserts = {+ R"({"id": "0"})"+ };++ nlohmann::json update_doc;+ auto import_response = coll1->add_many(upserts, update_doc, UPSERT);+ ASSERT_TRUE(import_response["success"].get<bool>());+ ASSERT_EQ(1, import_response["num_imported"].get<int>());++ auto results = coll1->search("*", {},+ "", {}, {}, {0}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ(1, results["hits"][0]["document"].size());++ ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().at("points")->size());+ ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().at("last_chance")->size());++ // put the original doc back+ ASSERT_TRUE(coll1->add(doc1.dump(), UPSERT).ok());++ results = coll1->search("*", {},+ "", {}, {}, {0}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(3, results["hits"][0]["document"].size());++ // update doc with missing fields: existing fields should NOT be removed++ upserts = {+ R"({"id": "0"})"+ };++ import_response = coll1->add_many(upserts, update_doc, UPDATE);+ ASSERT_TRUE(import_response["success"].get<bool>());+ ASSERT_EQ(1, import_response["num_imported"].get<int>());++ results = coll1->search("*", {},+ "", {}, {}, {0}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ(3, results["hits"][0]["document"].size());++ ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().at("points")->size());+ ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().at("last_chance")->size());+}+TEST_F(CollectionSpecificTest, FacetParallelizationVerification) {std::vector<field> fields = {field("name", field_types::STRING, false),field("category", field_types::STRING, true),