Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix bad document not being returned sometimes during import.

kishorenc249c4886dc31
diff --git a/src/collection.cpp b/src/collection.cpp
index f37800ec..4abfba74 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -206,17 +206,12 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines) {
nlohmann::json document;
Option<uint32_t> doc_seq_id_op = to_doc(json_line, document);
- const uint32_t seq_id = doc_seq_id_op.get();
+ const uint32_t seq_id = doc_seq_id_op.ok() ? doc_seq_id_op.get() : 0;
index_record record(i, seq_id, document);
// NOTE: we overwrite the input json_lines with result to avoid memory pressure
if(!doc_seq_id_op.ok()) {
- nlohmann::json index_res;
- index_res["error"] = doc_seq_id_op.error();
- index_res["success"] = false;
- index_res["document"] = json_line;
- json_lines[i] = index_res.dump();
record.index_failure(doc_seq_id_op.code(), doc_seq_id_op.error());
}
@@ -281,16 +276,14 @@ void Collection::batch_index(std::vector<std::vector<index_record>> &index_batch
// remove from in-memory store to keep the state synced
remove_document(index_record.document, index_record.seq_id, false);
}
- }
- if(index_record.indexed.ok()) {
json_out[index_record.position] = R"({"success": true})";
num_indexed++;
} else {
nlohmann::json res;
res["success"] = false;
res["error"] = index_record.indexed.error();
- res["document"] = index_record.document.dump();
+ res["document"] = json_out[index_record.position];
json_out[index_record.position] = res.dump();
}
}
@@ -1290,6 +1283,10 @@ void Collection::highlight_result(const field &search_field,
const Match & this_match = Match(field_order_kv->key, token_positions);
uint64_t this_match_score = this_match.get_match_score(1, field_order_kv->field_id);
match_indices.emplace_back(this_match, this_match_score, array_index);
+
+ /*LOG(INFO) << "doc_id: " << document["id"] << ", words_present: " << size_t(this_match.words_present)
+ << ", match_score: " << this_match_score
+ << ", match.distance: " << size_t(this_match.distance);*/
}
if(match_indices.empty()) {
diff --git a/src/index.cpp b/src/index.cpp
index 8a0821b2..21262d01 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1603,7 +1603,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
<< ", match_score: " << match_score
<< ", match.distance: " << match.distance
<< ", seq_id: " << seq_id << std::endl;
- std::cout << os.str();*/
+ LOG(INFO) << os.str();*/
}
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 91c55e56..dea95760 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1375,7 +1375,7 @@ TEST_F(CollectionTest, ImportDocuments) {
ASSERT_STREQ("Field `title` must be a string.", import_results[1]["error"].get<std::string>().c_str());
ASSERT_STREQ("Field `starring` has been declared in the schema, but is not found in the document.",
import_results[3]["error"].get<std::string>().c_str());
- ASSERT_STREQ("{\"cast\":[\"Tom Skerritt\"],\"id\":\"19\",\"points\":23,\"starring\":\"Jazz Gosh\",\"title\":123}",
+ ASSERT_STREQ("{\"title\": 123, \"starring\": \"Jazz Gosh\", \"points\": 23, \"cast\": [\"Tom Skerritt\"] }",
import_results[1]["document"].get<std::string>().c_str());
// record with duplicate IDs
@@ -1396,11 +1396,12 @@ TEST_F(CollectionTest, ImportDocuments) {
ASSERT_FALSE(import_results[1]["success"].get<bool>());
ASSERT_STREQ("A document with id id1 already exists.", import_results[1]["error"].get<std::string>().c_str());
- ASSERT_STREQ("{\"cast\":[\"Tom Skerritt\"],\"id\":\"id1\",\"points\":12,\"starring\":\"Rand Fish\",\"title\":\"Test1\"}",
- import_results[1]["document"].get<std::string>().c_str());
+ ASSERT_STREQ("{\"id\": \"id1\", \"title\": \"Test1\", \"starring\": \"Rand Fish\", \"points\": 12, "
+ "\"cast\": [\"Tom Skerritt\"] }",import_results[1]["document"].get<std::string>().c_str());
// handle bad import json
+ // valid JSON but not a document
more_records = {"[]"};
import_response = coll_mul_fields->add_many(more_records);
@@ -1414,6 +1415,22 @@ TEST_F(CollectionTest, ImportDocuments) {
ASSERT_STREQ("Bad JSON: not a properly formed document.", import_results[0]["error"].get<std::string>().c_str());
ASSERT_STREQ("[]", import_results[0]["document"].get<std::string>().c_str());
+ // invalid JSON
+ more_records = {"{"};
+ import_response = coll_mul_fields->add_many(more_records);
+
+ ASSERT_FALSE(import_response["success"].get<bool>());
+ ASSERT_EQ(0, import_response["num_imported"].get<int>());
+
+ import_results = import_res_to_json(more_records);
+ ASSERT_EQ(1, import_results.size());
+
+ ASSERT_EQ(false, import_results[0]["success"].get<bool>());
+ ASSERT_STREQ("Bad JSON: [json.exception.parse_error.101] parse error at line 1, column 2: syntax error "
+ "while parsing object key - unexpected end of input; expected string literal",
+ import_results[0]["error"].get<std::string>().c_str());
+ ASSERT_STREQ("{", import_results[0]["document"].get<std::string>().c_str());
+
collectionManager.drop_collection("coll_mul_fields");
}