Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle updates of repeating records within the same batch.

Kishore Nallanb14458efd487
diff --git a/src/collection.cpp b/src/collection.cpp
index f690e5f3..627454a8 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -223,22 +223,28 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
// NOTE: we overwrite the input json_lines with result to avoid memory pressure
record.is_update = false;
+ bool repeated_doc = false;
if(!doc_seq_id_op.ok()) {
record.index_failure(doc_seq_id_op.code(), doc_seq_id_op.error());
} else {
+ const std::string& doc_id = record.doc["id"].get<std::string>();
+ repeated_doc = (batch_doc_ids.find(doc_id) != batch_doc_ids.end());
+
+ if(repeated_doc) {
+ // when a document repeats, we send the batch until this document so that we can deal with conflicts
+ i--;
+ goto do_batched_index;
+ }
+
record.is_update = !doc_seq_id_op.get().is_new;
+
if(record.is_update) {
get_document_from_store(get_seq_id_key(seq_id), record.old_doc);
- } else {
- const std::string& doc_id = record.doc["id"].get<std::string>();
- if(batch_doc_ids.find(doc_id) != batch_doc_ids.end()) {
- record.index_failure(400, "Document with `id` " + doc_id + " already exists in the batch.");
- } else {
- batch_doc_ids.emplace(doc_id);
- }
}
+ batch_doc_ids.insert(doc_id);
+
// if `fallback_field_type` or `dynamic_fields` is enabled, update schema first before indexing
if(!fallback_field_type.empty() || !dynamic_fields.empty()) {
Option<bool> schema_change_op = check_and_update_schema(record.doc, dirty_values);
@@ -248,25 +254,11 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
}
}
- /*
- // check for memory threshold before allowing subsequent batches
- if(is_exceeding_memory_threshold()) {
- exceeds_memory_limit = true;
- }
-
- if(exceeds_memory_limit) {
- nlohmann::json index_res;
- index_res["error"] = "Max memory ratio exceeded.";
- index_res["success"] = false;
- index_res["document"] = json_line;
- json_lines[i] = index_res.dump();
- record.index_failure(500, "Max memory ratio exceeded.");
- }
- */
-
index_records.emplace_back(std::move(record));
- if((i+1) % index_batch_size == 0 || i == json_lines.size()-1) {
+ do_batched_index:
+
+ if((i+1) % index_batch_size == 0 || i == json_lines.size()-1 || repeated_doc) {
batch_index(index_records, json_lines, num_indexed);
// to return the document for the single doc add cases
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index c148b659..3c11518f 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -1452,7 +1452,7 @@ TEST_F(CollectionSpecificTest, ImportDocumentWithRepeatingIDInTheSameBatch) {
ASSERT_TRUE(nlohmann::json::parse(import_records[0])["success"].get<bool>());
ASSERT_FALSE(nlohmann::json::parse(import_records[1])["success"].get<bool>());
- ASSERT_EQ("Document with `id` 0 already exists in the batch.",
+ ASSERT_EQ("A document with id 0 already exists.",
nlohmann::json::parse(import_records[1])["error"].get<std::string>());
auto results = coll1->search("levis", {"name"},
@@ -1485,7 +1485,7 @@ TEST_F(CollectionSpecificTest, ImportDocumentWithRepeatingIDInTheSameBatch) {
ASSERT_TRUE(import_response["success"].get<bool>());
ASSERT_EQ(2, import_response["num_imported"].get<int>());
- // repeated ID is rejected even if the first ID is not indexed due to some error
+ // repeated ID is NOT rejected if the first ID is not indexed due to some error
import_records.clear();
doc1.erase("name");
doc1["id"] = "100";
@@ -1496,17 +1496,187 @@ TEST_F(CollectionSpecificTest, ImportDocumentWithRepeatingIDInTheSameBatch) {
import_response = coll1->add_many(import_records, document);
+ LOG(INFO) << import_response;
+
ASSERT_FALSE(import_response["success"].get<bool>());
- ASSERT_EQ(0, import_response["num_imported"].get<int>());
+ ASSERT_EQ(1, import_response["num_imported"].get<int>());
ASSERT_FALSE(nlohmann::json::parse(import_records[0])["success"].get<bool>());
- ASSERT_FALSE(nlohmann::json::parse(import_records[1])["success"].get<bool>());
- ASSERT_EQ("Document with `id` 100 already exists in the batch.",
- nlohmann::json::parse(import_records[1])["error"].get<std::string>());
+ ASSERT_EQ("Field `name` has been declared in the schema, but is not found in the document.",
+ nlohmann::json::parse(import_records[0])["error"].get<std::string>());
+
+ ASSERT_TRUE(nlohmann::json::parse(import_records[1])["success"].get<bool>());
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionSpecificTest, UpdateOfTwoDocsWithSameIdWithinSameBatch) {
+ std::vector<field> fields = {field("last_chance", field_types::BOOL, false, true),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+
+ // second update should reflect the result of first update
+ std::vector<std::string> updates = {
+ R"({"id": "0", "last_chance": false})",
+ R"({"id": "0", "points": 200})",
+ };
+
+ nlohmann::json update_doc;
+ auto import_response = coll1->add_many(updates, update_doc, UPDATE);
+ ASSERT_TRUE(import_response["success"].get<bool>());
+ ASSERT_EQ(2, import_response["num_imported"].get<int>());
+
+ auto results = coll1->search("*", {},
+ "", {}, {}, {0}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true).get();
+
+ LOG(INFO) << results;
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_TRUE(results["hits"][0]["document"].contains("last_chance"));
+ ASSERT_EQ(false, results["hits"][0]["document"]["last_chance"].get<bool>());
+
+ ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().at("points")->size());
+ ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().at("last_chance")->size());
+}
+
+TEST_F(CollectionSpecificTest, UpsertOfTwoDocsWithSameIdWithinSameBatch) {
+ std::vector<field> fields = {field("last_chance", field_types::BOOL, false, true),
+ field("points", field_types::INT32, false, true),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+
+ // first upsert removes both fields, so second upsert should only insert "points"
+ std::vector<std::string> upserts = {
+ R"({"id": "0", "last_chance": true})",
+ R"({"id": "0", "points": 200})",
+ };
+
+ nlohmann::json update_doc;
+ auto import_response = coll1->add_many(upserts, update_doc, UPSERT);
+ ASSERT_TRUE(import_response["success"].get<bool>());
+ ASSERT_EQ(2, import_response["num_imported"].get<int>());
+
+ auto results = coll1->search("*", {},
+ "", {}, {}, {0}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_TRUE(results["hits"][0]["document"].contains("points"));
+ ASSERT_FALSE(results["hits"][0]["document"].contains("last_chance"));
+ ASSERT_EQ(200, results["hits"][0]["document"]["points"].get<int32_t>());
+
+ ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().at("points")->size());
+ ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().at("last_chance")->size());
+
+ // update without doc id
+
+ upserts = {
+ R"({"last_chance": true})",
+ };
+
+ import_response = coll1->add_many(upserts, update_doc, UPDATE);
+ ASSERT_FALSE(import_response["success"].get<bool>());
+ ASSERT_EQ(0, import_response["num_imported"].get<int>());
+}
+
+TEST_F(CollectionSpecificTest, UpdateUpsertOfDocWithMissingFields) {
+ std::vector<field> fields = {field("last_chance", field_types::BOOL, false, true),
+ field("points", field_types::INT32, false, true),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "").get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["last_chance"] = true;
+ doc1["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+
+ // upsert doc with missing fields: should be removed from index
+ std::vector<std::string> upserts = {
+ R"({"id": "0"})"
+ };
+
+ nlohmann::json update_doc;
+ auto import_response = coll1->add_many(upserts, update_doc, UPSERT);
+ ASSERT_TRUE(import_response["success"].get<bool>());
+ ASSERT_EQ(1, import_response["num_imported"].get<int>());
+
+ auto results = coll1->search("*", {},
+ "", {}, {}, {0}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ(1, results["hits"][0]["document"].size());
+
+ ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().at("points")->size());
+ ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().at("last_chance")->size());
+
+ // put the original doc back
+ ASSERT_TRUE(coll1->add(doc1.dump(), UPSERT).ok());
+
+ results = coll1->search("*", {},
+ "", {}, {}, {0}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(3, results["hits"][0]["document"].size());
+
+ // update doc with missing fields: existing fields should NOT be removed
+
+ upserts = {
+ R"({"id": "0"})"
+ };
+
+ import_response = coll1->add_many(upserts, update_doc, UPDATE);
+ ASSERT_TRUE(import_response["success"].get<bool>());
+ ASSERT_EQ(1, import_response["num_imported"].get<int>());
+
+ results = coll1->search("*", {},
+ "", {}, {}, {0}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ(3, results["hits"][0]["document"].size());
+
+ ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().at("points")->size());
+ ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().at("last_chance")->size());
+}
+
TEST_F(CollectionSpecificTest, FacetParallelizationVerification) {
std::vector<field> fields = {field("name", field_types::STRING, false),
field("category", field_types::STRING, true),