Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add test for del doc scrubbing.
Kishore Nallan56778dc8478f
diff --git a/src/collection.cpp b/src/collection.cppindex 46fcbe9a..6f50e5be 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -211,6 +211,7 @@ void Collection::get_doc_changes(const nlohmann::json &document, nlohmann::jsonnew_doc[it.key()] = it.value();if(old_doc.count(it.key()) != 0) {// key exists in the stored doc, so it must be reindexed+ // we need to check for this because a field can be optionaldel_doc[it.key()] = old_doc[it.key()];}}diff --git a/src/index.cpp b/src/index.cppindex 8bf15ec2..29fe1a57 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -322,10 +322,12 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,}void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, nlohmann::json& old_doc) {- for (auto it = del_doc.begin(); it != del_doc.end(); ++it) {+ auto it = del_doc.cbegin();+ while(it != del_doc.cend()) {const std::string& field_name = it.key();const auto& search_field_it = search_schema.find(field_name);if(search_field_it == search_schema.end()) {+ ++it;continue;}@@ -338,6 +340,7 @@ void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_dotokenize_doc_field(old_doc, field_name, search_field, old_tokens);if(old_tokens.size() != reindex_tokens.size()) {+ ++it;continue;}@@ -353,7 +356,9 @@ void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_do}if(exact_match) {- del_doc.erase(field_name);+ it = del_doc.erase(it);+ } else {+ ++it;}}}diff --git a/test/index_test.cpp b/test/index_test.cppnew file mode 100644index 00000000..c069565e--- /dev/null+++ b/test/index_test.cpp@@ -0,0 +1,59 @@+#include <gtest/gtest.h>+#include "index.h"+#include <vector>++TEST(IndexTest, ScrubReindexDoc) {+ std::unordered_map<std::string, field> search_schema;+ search_schema.emplace("title", field("title", field_types::STRING, false));+ search_schema.emplace("points", field("title", field_types::INT32, false));+ search_schema.emplace("cast", field("cast", field_types::STRING_ARRAY, false));+ search_schema.emplace("movie", field("movie", field_types::BOOL, false));++ Index index("index", search_schema, {}, {});+ nlohmann::json old_doc;+ old_doc["id"] = "1";+ old_doc["title"] = "One more thing.";+ old_doc["points"] = 100;+ old_doc["cast"] = {"John Wick", "Jeremy Renner"};+ old_doc["movie"] = true;++ // all fields remain same++ nlohmann::json update_doc1, del_doc1;+ update_doc1 = old_doc;+ del_doc1 = old_doc;++ index.scrub_reindex_doc(update_doc1, del_doc1, old_doc);+ ASSERT_EQ(1, del_doc1.size());+ ASSERT_STREQ("1", del_doc1["id"].get<std::string>().c_str());++ // when only some fields are updated++ nlohmann::json update_doc2, del_doc2;+ update_doc2["id"] = "1";+ update_doc2["points"] = 100;+ update_doc2["cast"] = {"Jack"};++ del_doc2 = update_doc2;++ index.scrub_reindex_doc(update_doc2, del_doc2, old_doc);+ ASSERT_EQ(2, del_doc2.size());+ ASSERT_STREQ("1", del_doc2["id"].get<std::string>().c_str());+ std::vector<std::string> cast = del_doc2["cast"].get<std::vector<std::string>>();+ ASSERT_EQ(1, cast.size());+ ASSERT_STREQ("Jack", cast[0].c_str());++ // containing fields not part of search schema++ nlohmann::json update_doc3, del_doc3;+ update_doc3["id"] = "1";+ update_doc3["title"] = "The Lawyer";+ update_doc3["foo"] = "Bar";++ del_doc3 = update_doc3;+ index.scrub_reindex_doc(update_doc3, del_doc3, old_doc);+ ASSERT_EQ(3, del_doc3.size());+ ASSERT_STREQ("1", del_doc3["id"].get<std::string>().c_str());+ ASSERT_STREQ("The Lawyer", del_doc3["title"].get<std::string>().c_str());+ ASSERT_STREQ("Bar", del_doc3["foo"].get<std::string>().c_str());+}\ No newline at end of file