Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Return raw doc in response when indexing fails during import.

kishorencd42a04acb997
diff --git a/src/collection.cpp b/src/collection.cpp
index b4c5e3ee..4ebd6de3 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -206,6 +206,9 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines) {
nlohmann::json document;
Option<uint32_t> doc_seq_id_op = to_doc(json_line, document);
+ const uint32_t seq_id = doc_seq_id_op.get();
+ index_record record(i, seq_id, document);
+
// NOTE: we overwrite the input json_lines with result to avoid memory pressure
if(!doc_seq_id_op.ok()) {
@@ -213,9 +216,8 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines) {
index_res["error"] = doc_seq_id_op.error();
index_res["success"] = false;
index_res["document"] = json_line;
-
json_lines[i] = index_res.dump();
- continue;
+ record.index_failure(doc_seq_id_op.code(), doc_seq_id_op.error());
}
// check for memory threshold before allowing subsequent batches
@@ -228,13 +230,10 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines) {
index_res["error"] = "Max memory ratio exceeded.";
index_res["success"] = false;
index_res["document"] = json_line;
-
json_lines[i] = index_res.dump();
- continue;
+ record.index_failure(500, "Max memory ratio exceeded.");
}
- const uint32_t seq_id = doc_seq_id_op.get();
- index_record record(i, seq_id, document);
iter_batch[seq_id % this->get_num_memory_shards()].emplace_back(record);
if((i+1) % index_batch_size == 0 || i == json_lines.size()-1) {
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index f851c178..3eee6ddf 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1348,9 +1348,9 @@ TEST_F(CollectionTest, ImportDocuments) {
// record with duplicate IDs
- more_records = {"{\"id\": \"id1\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, "
+ more_records = {"{\"id\": \"id2\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, "
"\"cast\": [\"Tom Skerritt\"] }",
- "{\"id\": \"id2\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, "
+ "{\"id\": \"id1\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, "
"\"cast\": [\"Tom Skerritt\"] }"};
import_response = coll_mul_fields->add_many(more_records);
@@ -1360,12 +1360,12 @@ TEST_F(CollectionTest, ImportDocuments) {
import_results = import_res_to_json(more_records);
ASSERT_EQ(2, import_results.size());
- ASSERT_FALSE(import_results[0]["success"].get<bool>());
- ASSERT_TRUE(import_results[1]["success"].get<bool>());
+ ASSERT_TRUE(import_results[0]["success"].get<bool>());
+ ASSERT_FALSE(import_results[1]["success"].get<bool>());
- ASSERT_STREQ("A document with id id1 already exists.", import_results[0]["error"].get<std::string>().c_str());
- ASSERT_STREQ("{\"id\": \"id1\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, \"cast\": [\"Tom Skerritt\"] }",
- import_results[0]["document"].get<std::string>().c_str());
+ ASSERT_STREQ("A document with id id1 already exists.", import_results[1]["error"].get<std::string>().c_str());
+ ASSERT_STREQ("{\"cast\":[\"Tom Skerritt\"],\"id\":\"id1\",\"points\":12,\"starring\":\"Rand Fish\",\"title\":\"Test1\"}",
+ import_results[1]["document"].get<std::string>().c_str());
// handle bad import json