Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Optimize facet removal.
Kishore Nallan9872eebdf522
diff --git a/include/facet_index.h b/include/facet_index.hindex e5ae753f..910ead73 100644--- a/include/facet_index.h+++ b/include/facet_index.h@@ -114,6 +114,12 @@ private:// auto incrementing ID that is assigned to each unique facet value stringstd::atomic_uint32_t next_facet_id = 0;+ void get_stringified_value(const nlohmann::json& value, const field& afield,+ std::vector<std::string>& values);++ void get_stringified_values(const nlohmann::json& document, const field& afield,+ std::vector<std::string>& values);+public:facet_index_t() = default;@@ -127,7 +133,7 @@ public:void erase(const std::string& field_name);- void remove(const std::string& field_name, const uint32_t seq_id);+ void remove(const nlohmann::json& doc, const field& afield, const uint32_t seq_id);bool contains(const std::string& field_name);diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 9bc35caf..bcbaa8ac 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1872,8 +1872,6 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectionLOG(INFO) << "Indexed " << num_indexed_docs << "/" << num_found_docs<< " documents into collection " << collection->get_name();- collection->_get_index()->log_insert_time = true;- collection->_get_index()->_get_facet_index()->log_insert_time = true;return Option<bool>(true);}diff --git a/src/facet_index.cpp b/src/facet_index.cppindex 7e5a112f..87a7312e 100644--- a/src/facet_index.cpp+++ b/src/facet_index.cpp@@ -213,48 +213,97 @@ void facet_index_t::erase(const std::string& field_name) {facet_field_map.erase(field_name);}-void facet_index_t::remove(const std::string& field_name, const uint32_t seq_id) {- const auto facet_field_it = facet_field_map.find(field_name);- if(facet_field_it != facet_field_map.end()) {- auto& facet_index_map = facet_field_it->second.fvalue_seq_ids;- std::vector<std::string> dead_fvalues;-- for(auto facet_ids_seq_ids = facet_index_map.begin(); facet_ids_seq_ids != facet_index_map.end(); facet_ids_seq_ids++) {- void*& ids = facet_ids_seq_ids->second.seq_ids;- if(ids && ids_t::contains(ids, seq_id)) {- ids_t::erase(ids, seq_id);- auto& count_list = facet_field_it->second.counts;- auto curr = facet_ids_seq_ids->second.facet_count_it;- auto old_count = curr->count;- curr->count = ids_t::num_ids(ids);- auto new_count = curr->count;-- // move the node lower in the count list- auto& count_map = facet_field_it->second.count_map;- update_count_nodes(count_list, count_map, old_count, new_count, curr);-- if(ids_t::num_ids(ids) == 0) {- ids_t::destroy_list(ids);- dead_fvalues.push_back(facet_ids_seq_ids->first);-- //remove from int64 lookup map first- auto& fhash_int64_map = facet_field_it->second.fhash_to_int64_map;- uint32_t fhash = facet_ids_seq_ids->second.facet_id;- fhash_int64_map.erase(fhash);-- count_map.erase(new_count);- count_list.erase(curr);- }- }+void facet_index_t::get_stringified_value(const nlohmann::json& value, const field& afield,+ std::vector<std::string>& values) {+ if(afield.is_int32()) {+ int32_t raw_val = value.get<int32_t>();+ values.push_back(std::to_string(raw_val));+ }+ else if(afield.is_int64()) {+ int64_t raw_val = value.get<int64_t>();+ values.push_back(std::to_string(raw_val));+ }+ else if(afield.is_string()) {+ const std::string& raw_val = value.get<std::string>().substr(0, 100);+ values.push_back(raw_val);+ }+ else if(afield.is_float()) {+ float raw_val = value.get<float>();+ values.push_back(StringUtils::float_to_str(raw_val));+ }+ else if(afield.is_bool()) {+ bool raw_val = value.get<bool>();+ auto fhash = (uint32_t)raw_val;+ auto str_val = (raw_val == 1) ? "true" : "false";+ values.emplace_back(str_val);+ }+}++void facet_index_t::get_stringified_values(const nlohmann::json& document, const field& afield,+ std::vector<std::string>& values) {+ bool is_array = afield.is_array();++ if(!is_array) {+ return get_stringified_value(document[afield.name], afield, values);+ } else {+ const auto& field_values = document[afield.name];+ for(size_t i = 0; i < field_values.size(); i++) {+ get_stringified_value(field_values[i], afield, values);}+ }+}++void facet_index_t::remove(const nlohmann::json& doc, const field& afield, const uint32_t seq_id) {+ const auto facet_field_it = facet_field_map.find(afield.name);+ if(facet_field_it == facet_field_map.end()) {+ return ;+ }- for(auto& dead_fvalue: dead_fvalues) {- facet_index_map.erase(dead_fvalue);+ auto& facet_index_map = facet_field_it->second.fvalue_seq_ids;+ std::vector<std::string> dead_fvalues;+ std::vector<std::string> values;+ get_stringified_values(doc, afield, values);++ for(const auto& value: values) {+ auto fvalue_it = facet_index_map.find(value);+ if(fvalue_it == facet_index_map.end()) {+ continue;}- auto& seq_id_hashes = facet_field_it->second.seq_id_hashes;- seq_id_hashes->erase(seq_id);+ void*& ids = fvalue_it->second.seq_ids;+ if(ids && ids_t::contains(ids, seq_id)) {+ ids_t::erase(ids, seq_id);+ auto& count_list = facet_field_it->second.counts;+ auto curr = fvalue_it->second.facet_count_it;+ auto old_count = curr->count;+ curr->count = ids_t::num_ids(ids);+ auto new_count = curr->count;++ // move the node lower in the count list+ auto& count_map = facet_field_it->second.count_map;+ update_count_nodes(count_list, count_map, old_count, new_count, curr);++ if(ids_t::num_ids(ids) == 0) {+ ids_t::destroy_list(ids);+ dead_fvalues.push_back(fvalue_it->first);++ //remove from int64 lookup map first+ auto& fhash_int64_map = facet_field_it->second.fhash_to_int64_map;+ uint32_t fhash = fvalue_it->second.facet_id;+ fhash_int64_map.erase(fhash);++ count_map.erase(new_count);+ count_list.erase(curr);+ }+ }}++ for(auto& dead_fvalue: dead_fvalues) {+ facet_index_map.erase(dead_fvalue);+ }++ auto& seq_id_hashes = facet_field_it->second.seq_id_hashes;+ seq_id_hashes->erase(seq_id);}size_t facet_index_t::get_facet_count(const std::string& field_name) {diff --git a/src/index.cpp b/src/index.cppindex 5cdd6fcd..d22a0ae2 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -6460,7 +6460,7 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const}// remove facets- facet_index_v4->remove(field_name, seq_id);+ facet_index_v4->remove(document, search_field, seq_id);// remove sort fieldif(sort_index.count(field_name) != 0) {diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex 1ac287df..6fc020f7 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -2855,7 +2855,11 @@ TEST_F(CollectionFacetingTest, FhashInt64MapTest) {facet_index_v4.insert("visitors", fvalue_to_seq_ids, seq_id_to_fvalues);ASSERT_EQ(3, facet_index_v4.get_fhash_int64_map("visitors").size());- facet_index_v4.remove("visitors", 0);+ field visitorsf("visitors", field_types::INT64, true);+ nlohmann::json doc;+ doc["visitors"] = 227489798;++ facet_index_v4.remove(doc, visitorsf, 0);ASSERT_EQ(2, facet_index_v4.get_fhash_int64_map("visitors").size());fvalue_to_seq_ids.clear();diff --git a/test/facet_index_test.cpp b/test/facet_index_test.cppindex 2e897b28..1c141388 100644--- a/test/facet_index_test.cpp+++ b/test/facet_index_test.cpp@@ -1,7 +1,7 @@#include <gtest/gtest.h>#include "facet_index.h"-TEST(FacetIndexTest, FacetValueDeletion) {+TEST(FacetIndexTest, FacetValueDeletionString) {facet_index_t findex;std::unordered_map<facet_value_id_t, std::vector<uint32_t>, facet_value_id_t::Hash> fvalue_to_seq_ids;std::unordered_map<uint32_t, std::vector<facet_value_id_t>> seq_id_to_fvalues;@@ -13,12 +13,40 @@ TEST(FacetIndexTest, FacetValueDeletion) {seq_id_to_fvalues[1] = {nike};seq_id_to_fvalues[2] = {nike};- findex.insert("brands", fvalue_to_seq_ids, seq_id_to_fvalues, true);- findex.remove("nike", 0);- findex.remove("nike", 1);- findex.remove("nike", 2);+ field brandf("brand", field_types::STRING, true);+ nlohmann::json doc;+ doc["brand"] = "nike";- ASSERT_FALSE(findex.facet_value_exists("brands", "nike"));+ findex.insert("brand", fvalue_to_seq_ids, seq_id_to_fvalues, true);+ findex.remove(doc, brandf, 0);+ findex.remove(doc, brandf, 1);+ findex.remove(doc, brandf, 2);++ ASSERT_FALSE(findex.facet_value_exists("brand", "nike"));+}++TEST(FacetIndexTest, FacetValueDeletionFloat) {+ facet_index_t findex;+ std::unordered_map<facet_value_id_t, std::vector<uint32_t>, facet_value_id_t::Hash> fvalue_to_seq_ids;+ std::unordered_map<uint32_t, std::vector<facet_value_id_t>> seq_id_to_fvalues;++ facet_value_id_t price1("99.95", 1);++ fvalue_to_seq_ids[price1] = {0, 1, 2};+ seq_id_to_fvalues[0] = {price1};+ seq_id_to_fvalues[1] = {price1};+ seq_id_to_fvalues[2] = {price1};++ field pricef("price", field_types::FLOAT, true);+ nlohmann::json doc;+ doc["price"] = "99.95";++ findex.insert("brand", fvalue_to_seq_ids, seq_id_to_fvalues, true);+ findex.remove(doc, pricef, 0);+ findex.remove(doc, pricef, 1);+ findex.remove(doc, pricef, 2);++ ASSERT_FALSE(findex.facet_value_exists("price", "99.95"));}TEST(FacetIndexTest, UpdateWhenAllCountsLessThanNewCount) {