Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add test for del doc scrubbing.

Kishore Nallan56778dc8478f
diff --git a/src/collection.cpp b/src/collection.cpp
index 46fcbe9a..6f50e5be 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -211,6 +211,7 @@ void Collection::get_doc_changes(const nlohmann::json &document, nlohmann::json
new_doc[it.key()] = it.value();
if(old_doc.count(it.key()) != 0) {
// key exists in the stored doc, so it must be reindexed
+ // we need to check for this because a field can be optional
del_doc[it.key()] = old_doc[it.key()];
}
}
diff --git a/src/index.cpp b/src/index.cpp
index 8bf15ec2..29fe1a57 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -322,10 +322,12 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,
}
void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, nlohmann::json& old_doc) {
- for (auto it = del_doc.begin(); it != del_doc.end(); ++it) {
+ auto it = del_doc.cbegin();
+ while(it != del_doc.cend()) {
const std::string& field_name = it.key();
const auto& search_field_it = search_schema.find(field_name);
if(search_field_it == search_schema.end()) {
+ ++it;
continue;
}
@@ -338,6 +340,7 @@ void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_do
tokenize_doc_field(old_doc, field_name, search_field, old_tokens);
if(old_tokens.size() != reindex_tokens.size()) {
+ ++it;
continue;
}
@@ -353,7 +356,9 @@ void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_do
}
if(exact_match) {
- del_doc.erase(field_name);
+ it = del_doc.erase(it);
+ } else {
+ ++it;
}
}
}
diff --git a/test/index_test.cpp b/test/index_test.cpp
new file mode 100644
index 00000000..c069565e
--- /dev/null
+++ b/test/index_test.cpp
@@ -0,0 +1,59 @@
+#include <gtest/gtest.h>
+#include "index.h"
+#include <vector>
+
+TEST(IndexTest, ScrubReindexDoc) {
+ std::unordered_map<std::string, field> search_schema;
+ search_schema.emplace("title", field("title", field_types::STRING, false));
+ search_schema.emplace("points", field("title", field_types::INT32, false));
+ search_schema.emplace("cast", field("cast", field_types::STRING_ARRAY, false));
+ search_schema.emplace("movie", field("movie", field_types::BOOL, false));
+
+ Index index("index", search_schema, {}, {});
+ nlohmann::json old_doc;
+ old_doc["id"] = "1";
+ old_doc["title"] = "One more thing.";
+ old_doc["points"] = 100;
+ old_doc["cast"] = {"John Wick", "Jeremy Renner"};
+ old_doc["movie"] = true;
+
+ // all fields remain same
+
+ nlohmann::json update_doc1, del_doc1;
+ update_doc1 = old_doc;
+ del_doc1 = old_doc;
+
+ index.scrub_reindex_doc(update_doc1, del_doc1, old_doc);
+ ASSERT_EQ(1, del_doc1.size());
+ ASSERT_STREQ("1", del_doc1["id"].get<std::string>().c_str());
+
+ // when only some fields are updated
+
+ nlohmann::json update_doc2, del_doc2;
+ update_doc2["id"] = "1";
+ update_doc2["points"] = 100;
+ update_doc2["cast"] = {"Jack"};
+
+ del_doc2 = update_doc2;
+
+ index.scrub_reindex_doc(update_doc2, del_doc2, old_doc);
+ ASSERT_EQ(2, del_doc2.size());
+ ASSERT_STREQ("1", del_doc2["id"].get<std::string>().c_str());
+ std::vector<std::string> cast = del_doc2["cast"].get<std::vector<std::string>>();
+ ASSERT_EQ(1, cast.size());
+ ASSERT_STREQ("Jack", cast[0].c_str());
+
+ // containing fields not part of search schema
+
+ nlohmann::json update_doc3, del_doc3;
+ update_doc3["id"] = "1";
+ update_doc3["title"] = "The Lawyer";
+ update_doc3["foo"] = "Bar";
+
+ del_doc3 = update_doc3;
+ index.scrub_reindex_doc(update_doc3, del_doc3, old_doc);
+ ASSERT_EQ(3, del_doc3.size());
+ ASSERT_STREQ("1", del_doc3["id"].get<std::string>().c_str());
+ ASSERT_STREQ("The Lawyer", del_doc3["title"].get<std::string>().c_str());
+ ASSERT_STREQ("Bar", del_doc3["foo"].get<std::string>().c_str());
+}
\ No newline at end of file