Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Use batching for imports.

kishorenc237f47974a35
diff --git a/include/collection.h b/include/collection.h
index 69f15832..5c356bd9 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -152,7 +152,7 @@ private:
bool highlighted_fully,
highlight_t &highlight);
- void remove_document(nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);
+ void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);
void populate_overrides(std::string query,
const std::map<size_t, std::vector<std::string>>& pinned_hits,
@@ -221,7 +221,7 @@ public:
Option<nlohmann::json> add(const std::string & json_str);
- Option<nlohmann::json> add_many(const std::string & json_str);
+ Option<nlohmann::json> add_many(std::vector<std::string>& json_lines);
Option<nlohmann::json> search(const std::string & query, const std::vector<std::string> & search_fields,
const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
@@ -243,6 +243,8 @@ public:
Option<nlohmann::json> get(const std::string & id);
+ bool doc_exists(const std::string & id);
+
Option<std::string> remove(const std::string & id, bool remove_from_store = true);
Option<uint32_t> add_override(const override_t & override);
@@ -261,8 +263,7 @@ public:
Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id);
- void par_index_in_memory(std::vector<std::vector<index_record>> & iter_batch,
- batch_index_result & result);
+ size_t par_index_in_memory(std::vector<std::vector<index_record>> & iter_batch, std::vector<size_t>& indexed_counts);
static void prune_document(nlohmann::json &document, const spp::sparse_hash_set<std::string> & include_fields,
const spp::sparse_hash_set<std::string> & exclude_fields);
@@ -288,5 +289,8 @@ public:
void aggregate_topster(size_t query_index, Topster &topster, Topster *index_topster) const;
void populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs) const;
+
+ void batch_index(std::vector<std::vector<index_record>> &index_batches, std::vector<std::string>& json_out,
+ size_t &num_indexed);
};
diff --git a/include/collection_manager.h b/include/collection_manager.h
index fb08bccd..afaa0630 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -39,10 +39,10 @@ private:
~CollectionManager() = default;
- Option<std::string> get_first_index_error(const std::vector<index_result> & items) {
- for(const auto & item: items) {
- if(!item.index_op.ok()) {
- return Option<std::string>(item.index_op.error());
+ Option<std::string> get_first_index_error(const std::vector<index_record>& index_records) {
+ for(const auto & index_record: index_records) {
+ if(!index_record.indexed.ok()) {
+ return Option<std::string>(index_record.indexed.error());
}
}
diff --git a/include/index.h b/include/index.h
index 36cae022..cb01cf80 100644
--- a/include/index.h
+++ b/include/index.h
@@ -80,51 +80,23 @@ struct search_args {
};
struct index_record {
- size_t record_pos; // position of record in the original request
-
+ size_t position; // position of record in the original request
uint32_t seq_id;
- std::string json_str;
nlohmann::json document;
- index_record(size_t record_pos, uint32_t seq_id, const std::string & json_str, const nlohmann::json & doc):
- record_pos(record_pos), seq_id(seq_id), json_str(json_str), document(doc) {
-
- }
-};
-
-struct index_result {
- index_record record;
- Option<bool> index_op; // indicates if the indexing operation was a success
-
- index_result(const index_record & record, const Option<bool> & index_op):
- record(record), index_op(index_op) {
-
- }
-
- bool operator<(const index_result & a) const {
- return record.record_pos < a.record.record_pos;
- }
-};
-
-struct batch_index_result {
- std::vector<index_result> items;
- size_t num_indexed = 0;
+ Option<bool> indexed; // indicates if the indexing operation was a success
- batch_index_result() {
+ index_record(size_t record_pos, uint32_t seq_id, const nlohmann::json& doc):
+ position(record_pos), seq_id(seq_id), document(doc), indexed(true) {
}
- void failure(const index_record & record, const uint32_t err_code, const std::string & err_msg) {
- Option<bool> index_op_failure(err_code, err_msg);
- index_result res(record, index_op_failure);
- items.push_back(res);
+ void index_failure(const uint32_t err_code, const std::string & err_msg) {
+ indexed = Option<bool>(err_code, err_msg);
}
- void success(const index_record & record) {
- Option<bool> index_op_success(true);
- index_result res(record, index_op_success);
- items.push_back(res);
- num_indexed++;
+ void index_success(const index_record & record) {
+ indexed = Option<bool>(true);
}
};
@@ -252,7 +224,7 @@ public:
std::vector<std::vector<KV*>> & override_result_kvs,
const size_t typo_tokens_threshold);
- Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);
+ Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document);
art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);
@@ -276,7 +248,7 @@ public:
const std::unordered_map<std::string, field> & search_schema,
const std::map<std::string, field> & facet_schema);
- static batch_index_result batch_memory_index(Index *index,
+ static size_t batch_memory_index(Index *index,
std::vector<index_record> & iter_batch,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
diff --git a/src/collection.cpp b/src/collection.cpp
index e3724a11..b44e15ee 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -116,11 +116,8 @@ Option<uint32_t> Collection::to_doc(const std::string & json_str, nlohmann::json
return Option<uint32_t>(400, "Document's `id` field should be a string.");
}
- std::string doc_id = document["id"];
- Option<nlohmann::json> doc_option = get(doc_id);
-
- // we need to check if document ID already exists before attempting to index
- if(doc_option.ok()) {
+ const std::string& doc_id = document["id"];
+ if(doc_exists(doc_id)) {
return Option<uint32_t>(409, std::string("A document with id ") + doc_id + " already exists.");
}
@@ -180,21 +177,20 @@ Option<nlohmann::json> Collection::add(const std::string & json_str) {
return Option<nlohmann::json>(document);
}
-Option<nlohmann::json> Collection::add_many(const std::string & json_lines_str) {
- std::vector<std::string> json_lines;
- StringUtils::split(json_lines_str, json_lines, "\n");
-
+Option<nlohmann::json> Collection::add_many(std::vector<std::string>& json_lines) {
if(json_lines.empty()) {
return Option<nlohmann::json>(400, "The request body was empty. So, no records were imported.");
}
std::vector<std::vector<index_record>> iter_batch;
- batch_index_result result;
for(size_t i = 0; i < num_indices; i++) {
- iter_batch.push_back(std::vector<index_record>());
+ iter_batch.emplace_back(std::vector<index_record>());
}
+ const size_t index_batch_size = 1000;
+ size_t num_indexed = 0;
+
for(size_t i=0; i < json_lines.size(); i++) {
const std::string & json_line = json_lines[i];
@@ -202,59 +198,66 @@ Option<nlohmann::json> Collection::add_many(const std::string & json_lines_str)
Option<uint32_t> doc_seq_id_op = to_doc(json_line, document);
if(!doc_seq_id_op.ok()) {
- index_record record(i, 0, "", document);
- result.failure(record, doc_seq_id_op.code(), doc_seq_id_op.error());
+ nlohmann::json index_res;
+ index_res["error"] = doc_seq_id_op.error();
+ index_res["success"] = false;
+
+ // NOTE: we overwrite the input json_lines with result to avoid memory pressure
+ json_lines[i] = index_res.dump();
continue;
}
const uint32_t seq_id = doc_seq_id_op.get();
- index_record record(i, seq_id, json_line, document);
- iter_batch[seq_id % this->get_num_indices()].push_back(record);
- }
-
- par_index_in_memory(iter_batch, result);
+ index_record record(i, seq_id, document);
+ iter_batch[seq_id % this->get_num_indices()].emplace_back(record);
- std::sort(result.items.begin(), result.items.end());
-
- // store documents only documents that were indexed in-memory successfully
- for(index_result & item: result.items) {
- if(item.index_op.ok()) {
- rocksdb::WriteBatch batch;
- const std::string seq_id_str = std::to_string(item.record.seq_id);
-
- batch.Put(get_doc_id_key(item.record.document["id"]), seq_id_str);
- batch.Put(get_seq_id_key(item.record.seq_id), item.record.document.dump());
- bool write_ok = store->batch_write(batch);
-
- if(!write_ok) {
- Option<bool> index_op_failure(500, "Could not write to on-disk storage.");
- item.index_op = index_op_failure;
-
- // remove from in-memory store to keep the state synced
- remove_document(item.record.document, item.record.seq_id, false);
+ if((i+1) % index_batch_size == 0 || i == json_lines.size()-1) {
+ batch_index(iter_batch, json_lines, num_indexed);
+ for(size_t i_index = 0; i_index < get_num_indices(); i_index++) {
+ iter_batch[i_index].clear();
}
}
}
- nlohmann::json resp;
- resp["success"] = (result.num_indexed == json_lines.size());
- resp["num_imported"] = result.num_indexed;
+ nlohmann::json resp_summary;
+ resp_summary["num_imported"] = num_indexed;
+ resp_summary["success"] = (num_indexed == json_lines.size());
- resp["items"] = nlohmann::json::array();
- for(const index_result & item: result.items) {
- nlohmann::json item_obj;
+ return Option<nlohmann::json>(resp_summary);
+}
- if(!item.index_op.ok()) {
- item_obj["error"] = item.index_op.error();
- item_obj["success"] = false;
- } else {
- item_obj["success"] = true;
- }
+void Collection::batch_index(std::vector<std::vector<index_record>> &index_batches, std::vector<std::string>& json_out,
+ size_t &num_indexed) {
+ std::vector<size_t> indexed_counts;
+ indexed_counts.reserve(index_batches.size());
+ par_index_in_memory(index_batches, indexed_counts);
+
+ // store only documents that were indexed in-memory successfully
+ for(auto& index_batch: index_batches) {
+ for(auto& index_record: index_batch) {
+ if(index_record.indexed.ok()) {
+ const std::string& seq_id_str = std::to_string(index_record.seq_id);
+
+ rocksdb::WriteBatch batch;
+ batch.Put(get_doc_id_key(index_record.document["id"]), seq_id_str);
+ batch.Put(get_seq_id_key(index_record.seq_id), index_record.document.dump());
+ bool write_ok = store->batch_write(batch);
+
+ if(!write_ok) {
+ index_record.indexed = Option<bool>(500, "Could not write to on-disk storage.");;
+ // remove from in-memory store to keep the state synced
+ remove_document(index_record.document, index_record.seq_id, false);
+ }
+ }
- resp["items"].push_back(item_obj);
+ if(index_record.indexed.ok()) {
+ json_out[index_record.position] = R"({"success": true})";
+ num_indexed++;
+ } else {
+ json_out[index_record.position] = R"({"success": false, "error": ")" + index_record.indexed.error() + "\"}";
+ }
+ }
}
-
- return Option<nlohmann::json>(resp);
}
Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id) {
@@ -272,10 +275,10 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin
return Option<>(200);
}
-void Collection::par_index_in_memory(std::vector<std::vector<index_record>> & iter_batch,
- batch_index_result & result) {
-
- std::vector<std::future<batch_index_result>> futures;
+size_t Collection::par_index_in_memory(std::vector<std::vector<index_record>> & iter_batch,
+ std::vector<size_t>& indexed_counts) {
+ std::vector<std::future<size_t>> futures;
+ size_t num_indexed = 0;
for(size_t i=0; i < num_indices; i++) {
futures.push_back(
@@ -285,11 +288,13 @@ void Collection::par_index_in_memory(std::vector<std::vector<index_record>> & it
}
for(size_t i=0; i < futures.size(); i++) {
- batch_index_result future_res = futures[i].get();
- result.items.insert(result.items.end(), future_res.items.begin(), future_res.items.end());
- result.num_indexed += future_res.num_indexed;
- num_documents += future_res.num_indexed;
+ size_t num_indexed_future = futures[i].get();
+ num_documents += num_indexed_future;
+ num_indexed += num_indexed_future;
+ indexed_counts[i] = num_indexed_future;
}
+
+ return num_indexed;
}
void Collection::prune_document(nlohmann::json &document, const spp::sparse_hash_set<std::string>& include_fields,
@@ -1169,7 +1174,7 @@ void Collection::highlight_result(const field &search_field,
StringUtils & string_utils, size_t snippet_threshold,
bool highlighted_fully,
highlight_t & highlight) {
-
+
spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
std::vector<art_leaf *> query_suggestion;
@@ -1311,6 +1316,10 @@ void Collection::free_leaf_indices(spp::sparse_hash_map<const art_leaf *, uint32
}
}
+bool Collection::doc_exists(const std::string & id) {
+ return store->contains(get_doc_id_key(id));
+}
+
Option<nlohmann::json> Collection::get(const std::string & id) {
std::string seq_id_str;
StoreStatus seq_id_status = store->get(get_doc_id_key(id), seq_id_str);
@@ -1347,8 +1356,8 @@ Option<nlohmann::json> Collection::get(const std::string & id) {
return Option<nlohmann::json>(document);
}
-void Collection::remove_document(nlohmann::json & document, const uint32_t seq_id, bool remove_from_store) {
- std::string id = document["id"];
+void Collection::remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store) {
+ const std::string& id = document["id"];
Index* index = indices[seq_id % num_indices];
index->remove(seq_id, document);
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index df8b5f3e..f5f408d3 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -143,41 +143,42 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {
num_docs_read++;
- iter_batch[seq_id % collection->get_num_indices()].push_back(
- index_record(0, seq_id, iter->value().ToString(), document)
- );
+ iter_batch[seq_id % collection->get_num_indices()].emplace_back(index_record(0, seq_id, document));
+
+ // Peek and check for last record right here so that we handle batched indexing correctly
+ // Without doing this, the "last batch" would have to be indexed outside the loop.
+ iter->Next();
+ bool last_record = !(iter->Valid() && iter->key().starts_with(seq_id_prefix));
+
+ if((num_docs_read % init_batch_size == 0) || last_record) {
+ std::vector<size_t> indexed_counts;
+ indexed_counts.reserve(iter_batch.size());
+
+ collection->par_index_in_memory(iter_batch, indexed_counts);
- if(num_docs_read % init_batch_size == 0) {
- batch_index_result res;
- collection->par_index_in_memory(iter_batch, res);
for(size_t i = 0; i < collection->get_num_indices(); i++) {
+ size_t num_records = iter_batch[i].size();
+ size_t num_indexed = indexed_counts[i];
+
+ if(num_indexed != num_records) {
+ delete iter;
+ const Option<std::string> & index_error_op = get_first_index_error(iter_batch[i]);
+ if(index_error_op.ok()) {
+ return Option<bool>(false, index_error_op.get());
+ }
+ }
iter_batch[i].clear();
}
- if(res.num_indexed != res.items.size()) {
+ if(last_record) {
delete iter;
- const Option<std::string> & index_error_op = get_first_index_error(res.items);
- if(index_error_op.ok()) {
- return Option<bool>(false, index_error_op.get());
- }
+ iter = nullptr;
+ break;
}
}
-
- iter->Next();
- }
-
- delete iter;
-
- batch_index_result res;
- collection->par_index_in_memory(iter_batch, res);
-
- if(res.num_indexed != res.items.size()) {
- const Option<std::string> & index_error_op = get_first_index_error(res.items);
- if(index_error_op.ok()) {
- return Option<bool>(false, index_error_op.get());
- }
}
+ assert(iter == nullptr);
add_to_collections(collection);
}
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 06952a80..9d45645f 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -582,16 +582,31 @@ bool post_import_documents(http_req & req, http_res & res) {
return false;
}
- Option<nlohmann::json> res_op = collection->add_many(req.body);
+ std::vector<std::string> json_lines;
+ StringUtils::split(req.body, json_lines, "\n");
+
+ const Option<nlohmann::json>& res_op = collection->add_many(json_lines);
if(!res_op.ok()) {
res.set(res_op.code(), res_op.error());
return false;
}
- const nlohmann::json& result = res_op.get();
- res.set_200(result.dump());
- return result["success"].get<bool>();
+ std::stringstream ss;
+ const std::string& import_summary_json = res_op.get().dump();
+
+ ss << import_summary_json << "\n";
+
+ for (size_t i = 0; i < json_lines.size(); i++) {
+ if(i == json_lines.size()-1) {
+ ss << json_lines[i];
+ } else {
+ ss << json_lines[i] << "\n";
+ }
+ }
+
+ res.set_200(ss.str());
+ return true;
}
bool get_fetch_document(http_req & req, http_res & res) {
@@ -981,6 +996,9 @@ bool async_write_request(void *data) {
if(!async_call && index_arg->req->_req != nullptr) {
// we have to return a response to the client
server->send_response(index_arg->req, index_arg->res);
+ } else {
+ delete index_arg->req;
+ delete index_arg->res;
}
if(index_arg->promise != nullptr) {
diff --git a/src/index.cpp b/src/index.cpp
index 66b0a299..5048e7d1 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -313,38 +313,34 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,
return Option<>(200);
}
-batch_index_result Index::batch_memory_index(Index *index, std::vector<index_record> & iter_batch,
- const std::string & default_sorting_field,
- const std::unordered_map<std::string, field> & search_schema,
- const std::map<std::string, field> & facet_schema) {
+size_t Index::batch_memory_index(Index *index, std::vector<index_record> & iter_batch,
+ const std::string & default_sorting_field,
+ const std::unordered_map<std::string, field> & search_schema,
+ const std::map<std::string, field> & facet_schema) {
- batch_index_result result;
+ size_t num_indexed = 0;
for(auto & index_rec: iter_batch) {
- if(index_rec.json_str.empty()) {
- // indicates bad record (upstream validation failure)
- continue;
- }
-
Option<uint32_t> validation_op = validate_index_in_memory(index_rec.document, index_rec.seq_id,
default_sorting_field,
search_schema, facet_schema);
if(!validation_op.ok()) {
- result.failure(index_rec, validation_op.code(), validation_op.error());
+ index_rec.index_failure(validation_op.code(), validation_op.error());
continue;
}
Option<uint32_t> index_mem_op = index->index_in_memory(index_rec.document, index_rec.seq_id, default_sorting_field);
if(!index_mem_op.ok()) {
- result.failure(index_rec, index_mem_op.code(), index_mem_op.error());
+ index_rec.index_failure(index_mem_op.code(), index_mem_op.error());
continue;
}
- result.success(index_rec);
+ index_rec.index_success(index_rec);
+ num_indexed++;
}
- return result;
+ return num_indexed;
}
void Index::insert_doc(const uint32_t score, art_tree *t, uint32_t seq_id,
@@ -1696,7 +1692,7 @@ void Index::remove_and_shift_offset_index(sorted_array &offset_index, const uint
delete[] new_array;
}
-Option<uint32_t> Index::remove(const uint32_t seq_id, nlohmann::json & document) {
+Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & document) {
for(auto & name_field: search_schema) {
if(name_field.second.optional && document.count(name_field.first) == 0) {
continue;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 12748830..df6c2d36 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1254,15 +1254,27 @@ TEST_F(CollectionTest, FilterOnFloatFields) {
collectionManager.drop_collection("coll_array_fields");
}
+std::vector<nlohmann::json> import_res_to_json(std::vector<std::string> imported_results) {
+ std::vector<nlohmann::json> out;
+
+ for(const auto& imported_result: imported_results) {
+ out.emplace_back(nlohmann::json::parse(imported_result));
+ }
+
+ return out;
+}
+
TEST_F(CollectionTest, ImportDocuments) {
Collection *coll_mul_fields;
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
std::stringstream strstream;
strstream << infile.rdbuf();
- std::string import_records = strstream.str();
infile.close();
+ std::vector<std::string> import_records;
+ StringUtils::split(strstream.str(), import_records, "\n");
+
std::vector<field> fields = {
field("title", field_types::STRING, false),
field("starring", field_types::STRING, false),
@@ -1282,7 +1294,6 @@ TEST_F(CollectionTest, ImportDocuments) {
nlohmann::json import_response = import_res.get();
ASSERT_TRUE(import_response["success"].get<bool>());
ASSERT_EQ(18, import_response["num_imported"].get<int>());
- ASSERT_EQ(0, import_response.count("errors"));
// now try searching for records
@@ -1304,19 +1315,20 @@ TEST_F(CollectionTest, ImportDocuments) {
}
// verify that empty import is caught gracefully
- import_res = coll_mul_fields->add_many("");
+ std::vector<std::string> empty_records;
+ import_res = coll_mul_fields->add_many(empty_records);
ASSERT_FALSE(import_res.ok());
ASSERT_STREQ("The request body was empty. So, no records were imported.", import_res.error().c_str());
// verify that only bad records are rejected, rest must be imported (records 2 and 4 are bad)
- std::string more_records = std::string("{\"id\": \"id1\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, "
- "\"cast\": [\"Tom Skerritt\"] }\n") +
- "{\"title\": 123, \"starring\": \"Jazz Gosh\", \"points\": 23, "
- "\"cast\": [\"Tom Skerritt\"] }\n" +
+ std::vector<std::string> more_records = {"{\"id\": \"id1\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, "
+ "\"cast\": [\"Tom Skerritt\"] }",
+ "{\"title\": 123, \"starring\": \"Jazz Gosh\", \"points\": 23, "
+ "\"cast\": [\"Tom Skerritt\"] }",
"{\"title\": \"Test3\", \"starring\": \"Brad Fin\", \"points\": 11, "
- "\"cast\": [\"Tom Skerritt\"] }\n" +
+ "\"cast\": [\"Tom Skerritt\"] }",
"{\"title\": \"Test4\", \"points\": 55, "
- "\"cast\": [\"Tom Skerritt\"] }\n";
+ "\"cast\": [\"Tom Skerritt\"] }"};
import_res = coll_mul_fields->add_many(more_records);
ASSERT_TRUE(import_res.ok());
@@ -1324,23 +1336,25 @@ TEST_F(CollectionTest, ImportDocuments) {
import_response = import_res.get();
ASSERT_FALSE(import_response["success"].get<bool>());
ASSERT_EQ(2, import_response["num_imported"].get<int>());
- ASSERT_EQ(4, import_response["items"].size());
- ASSERT_TRUE(import_response["items"][0]["success"].get<bool>());
- ASSERT_FALSE(import_response["items"][1]["success"].get<bool>());
- ASSERT_TRUE(import_response["items"][2]["success"].get<bool>());
- ASSERT_FALSE(import_response["items"][3]["success"].get<bool>());
+ std::vector<nlohmann::json> import_results = import_res_to_json(more_records);
- ASSERT_STREQ("Field `title` must be a string.", import_response["items"][1]["error"].get<std::string>().c_str());
+ ASSERT_EQ(4, import_results.size());
+ ASSERT_TRUE(import_results[0]["success"].get<bool>());
+ ASSERT_FALSE(import_results[1]["success"].get<bool>());
+ ASSERT_TRUE(import_results[2]["success"].get<bool>());
+ ASSERT_FALSE(import_results[3]["success"].get<bool>());
+
+ ASSERT_STREQ("Field `title` must be a string.", import_results[1]["error"].get<std::string>().c_str());
ASSERT_STREQ("Field `starring` has been declared in the schema, but is not found in the document.",
- import_response["items"][3]["error"].get<std::string>().c_str());
+ import_results[3]["error"].get<std::string>().c_str());
// record with duplicate IDs
- more_records = std::string("{\"id\": \"id1\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, "
- "\"cast\": [\"Tom Skerritt\"] }\n") +
+ more_records = {"{\"id\": \"id1\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, "
+ "\"cast\": [\"Tom Skerritt\"] }",
"{\"id\": \"id2\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, "
- "\"cast\": [\"Tom Skerritt\"] }";
+ "\"cast\": [\"Tom Skerritt\"] }"};
import_res = coll_mul_fields->add_many(more_records);
ASSERT_TRUE(import_res.ok());
@@ -1350,14 +1364,16 @@ TEST_F(CollectionTest, ImportDocuments) {
ASSERT_FALSE(import_response["success"].get<bool>());
ASSERT_EQ(1, import_response["num_imported"].get<int>());
- ASSERT_FALSE(import_response["items"][0]["success"].get<bool>());
- ASSERT_TRUE(import_response["items"][1]["success"].get<bool>());
+ import_results = import_res_to_json(more_records);
+ ASSERT_EQ(2, import_results.size());
+ ASSERT_FALSE(import_results[0]["success"].get<bool>());
+ ASSERT_TRUE(import_results[1]["success"].get<bool>());
- ASSERT_STREQ("A document with id id1 already exists.", import_response["items"][0]["error"].get<std::string>().c_str());
+ ASSERT_STREQ("A document with id id1 already exists.", import_results[0]["error"].get<std::string>().c_str());
// handle bad import json
- more_records = std::string("[]");
+ more_records = {"[]"};
import_res = coll_mul_fields->add_many(more_records);
ASSERT_TRUE(import_res.ok());
@@ -1365,8 +1381,10 @@ TEST_F(CollectionTest, ImportDocuments) {
ASSERT_FALSE(import_response["success"].get<bool>());
ASSERT_EQ(0, import_response["num_imported"].get<int>());
- ASSERT_EQ(1, import_response["items"].size());
- ASSERT_STREQ("Bad JSON.", import_response["items"][0]["error"].get<std::string>().c_str());
+
+ import_results = import_res_to_json(more_records);
+ ASSERT_EQ(1, import_results.size());
+ ASSERT_STREQ("Bad JSON.", import_results[0]["error"].get<std::string>().c_str());
collectionManager.drop_collection("coll_mul_fields");
}