Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix bad document not being returned sometimes during import.
kishorenc249c4886dc31
diff --git a/src/collection.cpp b/src/collection.cppindex f37800ec..4abfba74 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -206,17 +206,12 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines) {nlohmann::json document;Option<uint32_t> doc_seq_id_op = to_doc(json_line, document);- const uint32_t seq_id = doc_seq_id_op.get();+ const uint32_t seq_id = doc_seq_id_op.ok() ? doc_seq_id_op.get() : 0;index_record record(i, seq_id, document);// NOTE: we overwrite the input json_lines with result to avoid memory pressureif(!doc_seq_id_op.ok()) {- nlohmann::json index_res;- index_res["error"] = doc_seq_id_op.error();- index_res["success"] = false;- index_res["document"] = json_line;- json_lines[i] = index_res.dump();record.index_failure(doc_seq_id_op.code(), doc_seq_id_op.error());}@@ -281,16 +276,14 @@ void Collection::batch_index(std::vector<std::vector<index_record>> &index_batch// remove from in-memory store to keep the state syncedremove_document(index_record.document, index_record.seq_id, false);}- }- if(index_record.indexed.ok()) {json_out[index_record.position] = R"({"success": true})";num_indexed++;} else {nlohmann::json res;res["success"] = false;res["error"] = index_record.indexed.error();- res["document"] = index_record.document.dump();+ res["document"] = json_out[index_record.position];json_out[index_record.position] = res.dump();}}@@ -1290,6 +1283,10 @@ void Collection::highlight_result(const field &search_field,const Match & this_match = Match(field_order_kv->key, token_positions);uint64_t this_match_score = this_match.get_match_score(1, field_order_kv->field_id);match_indices.emplace_back(this_match, this_match_score, array_index);++ /*LOG(INFO) << "doc_id: " << document["id"] << ", words_present: " << size_t(this_match.words_present)+ << ", match_score: " << this_match_score+ << ", match.distance: " << size_t(this_match.distance);*/}if(match_indices.empty()) {diff --git a/src/index.cpp b/src/index.cppindex 8a0821b2..21262d01 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1603,7 +1603,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16<< ", match_score: " << match_score<< ", match.distance: " << match.distance<< ", seq_id: " << seq_id << std::endl;- std::cout << os.str();*/+ LOG(INFO) << os.str();*/}}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 91c55e56..dea95760 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1375,7 +1375,7 @@ TEST_F(CollectionTest, ImportDocuments) {ASSERT_STREQ("Field `title` must be a string.", import_results[1]["error"].get<std::string>().c_str());ASSERT_STREQ("Field `starring` has been declared in the schema, but is not found in the document.",import_results[3]["error"].get<std::string>().c_str());- ASSERT_STREQ("{\"cast\":[\"Tom Skerritt\"],\"id\":\"19\",\"points\":23,\"starring\":\"Jazz Gosh\",\"title\":123}",+ ASSERT_STREQ("{\"title\": 123, \"starring\": \"Jazz Gosh\", \"points\": 23, \"cast\": [\"Tom Skerritt\"] }",import_results[1]["document"].get<std::string>().c_str());// record with duplicate IDs@@ -1396,11 +1396,12 @@ TEST_F(CollectionTest, ImportDocuments) {ASSERT_FALSE(import_results[1]["success"].get<bool>());ASSERT_STREQ("A document with id id1 already exists.", import_results[1]["error"].get<std::string>().c_str());- ASSERT_STREQ("{\"cast\":[\"Tom Skerritt\"],\"id\":\"id1\",\"points\":12,\"starring\":\"Rand Fish\",\"title\":\"Test1\"}",- import_results[1]["document"].get<std::string>().c_str());+ ASSERT_STREQ("{\"id\": \"id1\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, "+ "\"cast\": [\"Tom Skerritt\"] }",import_results[1]["document"].get<std::string>().c_str());// handle bad import json+ // valid JSON but not a documentmore_records = {"[]"};import_response = coll_mul_fields->add_many(more_records);@@ -1414,6 +1415,22 @@ TEST_F(CollectionTest, ImportDocuments) {ASSERT_STREQ("Bad JSON: not a properly formed document.", import_results[0]["error"].get<std::string>().c_str());ASSERT_STREQ("[]", import_results[0]["document"].get<std::string>().c_str());+ // invalid JSON+ more_records = {"{"};+ import_response = coll_mul_fields->add_many(more_records);++ ASSERT_FALSE(import_response["success"].get<bool>());+ ASSERT_EQ(0, import_response["num_imported"].get<int>());++ import_results = import_res_to_json(more_records);+ ASSERT_EQ(1, import_results.size());++ ASSERT_EQ(false, import_results[0]["success"].get<bool>());+ ASSERT_STREQ("Bad JSON: [json.exception.parse_error.101] parse error at line 1, column 2: syntax error "+ "while parsing object key - unexpected end of input; expected string literal",+ import_results[0]["error"].get<std::string>().c_str());+ ASSERT_STREQ("{", import_results[0]["document"].get<std::string>().c_str());+collectionManager.drop_collection("coll_mul_fields");}