Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix facet counter list reordering.
Kishore Nallanc0a507750ad8
diff --git a/include/facet_index.h b/include/facet_index.hindex fb3ce3b7..998471c0 100644--- a/include/facet_index.h+++ b/include/facet_index.h@@ -80,6 +80,7 @@ private:struct facet_doc_ids_list_t {std::map<std::string, facet_id_seq_ids_t> fvalue_seq_ids;std::list<facet_count_t> counts;+ std::map<uint32_t, std::list<facet_count_t>::iterator> count_map;posting_list_t* seq_id_hashes = nullptr;spp::sparse_hash_map<uint32_t, int64_t> fhash_to_int64_map;@@ -154,4 +155,9 @@ public://get fhash=>int64 map for statsconst spp::sparse_hash_map<uint32_t, int64_t>& get_fhash_int64_map(const std::string& field_name);++ void update_count_nodes(std::list<facet_count_t>& count_list,+ std::map<uint32_t, std::list<facet_count_t>::iterator>& count_map,+ uint32_t old_count, uint32_t new_count,+ std::list<facet_count_t>::iterator& curr) const;};\ No newline at end of filediff --git a/src/facet_index.cpp b/src/facet_index.cppindex 3ac86418..119f6e7c 100644--- a/src/facet_index.cpp+++ b/src/facet_index.cpp@@ -2,6 +2,7 @@#include <tokenizer.h>#include "string_utils.h"#include "array_utils.h"+#include "bazel-src/external/onnx_runtime/cmake/external/protobuf/src/google/protobuf/map.h"void facet_index_t::initialize(const std::string& field) {const auto facet_field_map_it = facet_field_map.find(field);@@ -59,9 +60,31 @@ void facet_index_t::insert(const std::string& field_name,std::unordered_map<facefis.facet_id = facet_id;if(facet_index.has_value_index) {- count_list.emplace_back(fvalue.facet_value, seq_ids.size(), facet_id);- fis.facet_count_it = std::prev(count_list.end());+ auto new_count = seq_ids.size();fis.seq_ids = ids_t::create(seq_ids);++ auto& count_map = facet_index.count_map;+ auto count_map_it = count_map.lower_bound(new_count);++ count_list.emplace_front(fvalue.facet_value, new_count, facet_id);+ fis.facet_count_it = count_list.begin();+ auto curr = fis.facet_count_it;++ if(count_map_it == facet_index.count_map.end()) {+ // no element >= new_count, so it remains at head of list+ facet_index.count_map.emplace(curr->count, curr);+ } else if(count_map_it->second->count == new_count) {+ // `curr` must be placed _before_ anchor node+ // a) [7], 10, <7>, 3+ auto& existing_node = count_map_it->second;+ count_list.splice(existing_node, count_list, curr);+ } else {+ // b) [9], 12, 10, <10>, 7, 3+ // `curr` placed _after_ anchor node+ auto& gt_node = count_map_it->second;+ count_list.splice(std::next(gt_node), count_list, curr);+ count_map.emplace(curr->count, curr);+ }}fvalue_index.emplace(fvalue.facet_value, fis);@@ -73,12 +96,14 @@ void facet_index_t::insert(const std::string& field_name,std::unordered_map<faceauto facet_count_it = fvalue_index_it->second.facet_count_it;if(facet_count_it->facet_id == facet_id) {+ auto& count_map = facet_index.count_map;+ auto old_count = facet_count_it->count;facet_count_it->count = ids_t::num_ids(fvalue_index_it->second.seq_ids);+ auto new_count = facet_count_it->count;++ // Relocate `curr` to a new sorted location within count list:auto curr = facet_count_it;- while (curr != count_list.begin() && std::prev(curr)->count < curr->count) {- count_list.splice(curr, count_list, std::prev(curr)); // swaps list nodes- curr--;- }+ update_count_nodes(count_list, count_map, old_count, new_count, curr);} else {LOG(ERROR) << "Wrong reference stored for facet " << fvalue.facet_value << " with facet_id " << facet_id;}@@ -93,6 +118,79 @@ void facet_index_t::insert(const std::string& field_name,std::unordered_map<face}}+void facet_index_t::update_count_nodes(std::list<facet_count_t>& count_list,+ std::map<uint32_t, std::list<facet_count_t>::iterator>& count_map,+ uint32_t old_count, uint32_t new_count,+ std::list<facet_count_t>::iterator& curr) const {++ auto count_map_it = count_map.lower_bound(new_count);++ if(count_map_it == count_map.end()) {+ // all elements are < new_count++ // 5, 4, [4 -> 7]+ // 5, 4, [4 -> 7], 3+ // 5, [4 -> 7]++ // delete count map entry if `curr` is the anchor for `old_count`+ if(std::next(curr) == count_list.end() || std::next(curr)->count < old_count) {+ count_map.erase(old_count);++ // find a replacement for orig_count+ if(std::prev(curr)->count == old_count) {+ count_map.emplace(old_count, std::prev(curr));+ }+ }++ // `curr` placed at head of the count list+ count_list.splice(count_list.begin(), count_list, curr);+ count_map.emplace(new_count, curr);+ } else if(count_map_it->first == new_count) {+ // entry for new_count already exists in count map+ // a) 10, 7, [5 -> 7], 3+ // 10, 7, 5, [5 -> 7]+ // 10, 7, [5 -> 7]+ // 10, 7, [5 -> 7], 5++ auto existing_node = count_map_it->second;++ // delete count map entry if `curr` is the anchor for `old_count`+ if(std::next(curr) == count_list.end() || std::next(curr)->count < old_count) {+ count_map.erase(old_count);++ // find a replacement for orig_count+ if(std::prev(curr)->count == old_count) {+ count_map.emplace(old_count, std::prev(curr));+ }+ }++ // `curr` placed before anchor node+ count_list.splice(existing_node, count_list, curr);+ } else {+ // b) 10, 7, [7 -> 9], 3+ // 10, 7, [7 -> 9]+ // 10, [7 -> 9]+ // 10, [7 -> 9], 7+ // 10, 7, [5 -> 9], 3++ auto gt_node = count_map_it->second;++ // delete old entry if `orig_count` iterator is same as `curr`+ if(std::next(curr) == count_list.end() || std::next(curr)->count < old_count) {+ count_map.erase(old_count);++ // find a replacement for orig_count+ if(std::prev(curr)->count == old_count) {+ count_map.emplace(old_count, std::prev(curr));+ }+ }++ // `curr` placed after anchor node+ count_list.splice(std::next(gt_node), count_list, curr);+ count_map.emplace(new_count, curr);+ }+}+bool facet_index_t::contains(const std::string& field_name) {const auto& facet_field_it = facet_field_map.find(field_name);if(facet_field_it == facet_field_map.end()) {@@ -117,7 +215,14 @@ void facet_index_t::remove(const std::string& field_name, const uint32_t seq_id)if(ids && ids_t::contains(ids, seq_id)) {ids_t::erase(ids, seq_id);auto& count_list = facet_field_it->second.counts;- facet_ids_seq_ids->second.facet_count_it->count--;+ auto curr = facet_ids_seq_ids->second.facet_count_it;+ auto old_count = curr->count;+ curr->count--;+ auto new_count = curr->count;++ // move the node lower in the count list+ auto& count_map = facet_field_it->second.count_map;+ update_count_nodes(count_list, count_map, old_count, new_count, curr);if(ids_t::num_ids(ids) == 0) {ids_t::destroy_list(ids);@@ -129,7 +234,9 @@ void facet_index_t::remove(const std::string& field_name, const uint32_t seq_id)uint32_t fhash = facet_ids_seq_ids->second.facet_id;fhash_int64_map.erase(fhash);- count_list.erase(facet_ids_seq_ids->second.facet_count_it);+ count_map.erase(new_count);+ count_list.erase(curr);+auto node = facet_index_map.extract(facet_ids_seq_ids->first);node.key() = dead_fvalue;facet_index_map.insert(std::move(node));diff --git a/test/collection_optimized_faceting_test.cpp b/test/collection_optimized_faceting_test.cppindex 9a9f76fa..95a80eec 100644--- a/test/collection_optimized_faceting_test.cpp+++ b/test/collection_optimized_faceting_test.cpp@@ -2195,14 +2195,144 @@ TEST_F(CollectionOptimizedFacetingTest, ValueIndexStatsMinMax) {ASSERT_EQ(1, results["facet_counts"].size());ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());- ASSERT_EQ("9.3", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("9", results["facet_counts"][0]["counts"][0]["value"]);ASSERT_EQ("9.2", results["facet_counts"][0]["counts"][1]["value"]);//statsASSERT_EQ(5, results["facet_counts"][0]["stats"].size());- ASSERT_FLOAT_EQ(9.25, results["facet_counts"][0]["stats"]["avg"].get<double>());+ ASSERT_FLOAT_EQ(9.1, results["facet_counts"][0]["stats"]["avg"].get<double>());ASSERT_FLOAT_EQ(8.800000190734863, results["facet_counts"][0]["stats"]["min"].get<double>());ASSERT_FLOAT_EQ(9.300000190734863, results["facet_counts"][0]["stats"]["max"].get<double>());- ASSERT_FLOAT_EQ(18.5, results["facet_counts"][0]["stats"]["sum"].get<double>());+ ASSERT_FLOAT_EQ(18.2, results["facet_counts"][0]["stats"]["sum"].get<double>());ASSERT_FLOAT_EQ(2, results["facet_counts"][0]["stats"]["total_values"].get<size_t>());-}\ No newline at end of file+}++TEST_F(CollectionOptimizedFacetingTest, StringFacetsCountListOrderTest) {+ //check if count list is ordering facets+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, true),+ field("rating", field_types::FLOAT, false)};++ std::vector<sort_by> sort_fields = {sort_by("rating", "DESC")};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 4, fields, "rating").get();+ }++ std::vector<std::string> titles {"The Shawshank Redemption", "The Godfather", "The Dark Knight"};+ nlohmann::json doc;+ int i = 0;+ for(; i < 6; ++i) {+ doc["id"] = std::to_string(i);+ doc["title"] = titles[i%3];+ doc["rating"] = 8.5;+ coll1->add(doc.dump());+ }++ //add last title more+ for(; i < 10; ++i) {+ doc["id"] = std::to_string(i);+ doc["title"] = titles[2];+ doc["rating"] = 8.5;+ coll1->add(doc.dump());+ }++ std::vector<std::string> facets = {"title"};++ //limit max facets to 2+ nlohmann::json results = coll1->search("*", {"title"}, "", facets, sort_fields, {0}, 10, 1,+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 2,"", 30UL, 4UL,+ "", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},+ 4294967295UL, true, false, true, "", false, 6000000UL, 4UL,+ 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();++ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("The Dark Knight", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ(6, results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_EQ("The Godfather", results["facet_counts"][0]["counts"][1]["value"]);+ ASSERT_EQ(2, results["facet_counts"][0]["counts"][1]["count"]);+}++TEST_F(CollectionOptimizedFacetingTest, StringFacetsCountListRemoveTest) {+ //delete records and check if counts are updated++ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, true),+ field("rating", field_types::FLOAT, false)};++ std::vector<sort_by> sort_fields = {sort_by("rating", "DESC")};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 4, fields, "rating").get();+ }++ std::vector<std::string> titles {"The Shawshank Redemption", "The Godfather", "The Dark Knight"};+ nlohmann::json doc;+ int i = 0;+ for(; i < 6; ++i) {+ doc["id"] = std::to_string(i);+ doc["title"] = titles[i%3];+ doc["rating"] = 8.5;+ coll1->add(doc.dump());+ }++ //add last title more+ for(; i < 10; ++i) {+ doc["id"] = std::to_string(i);+ doc["title"] = titles[2];+ doc["rating"] = 8.5;+ coll1->add(doc.dump());+ }++ // remove first doc+ coll1->remove("0");++ std::vector<std::string> facets = {"title"};++ // limit max facets to 2+ nlohmann::json results = coll1->search("*", {"title"}, "", facets, sort_fields, {0}, 10, 1,+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 2,"", 30UL, 4UL,+ "", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},+ 4294967295UL, true, false, true, "", false, 6000000UL, 4UL,+ 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();++ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("The Dark Knight", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ(6, results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_EQ("The Godfather", results["facet_counts"][0]["counts"][1]["value"]);+ ASSERT_EQ(2, results["facet_counts"][0]["counts"][1]["count"]);++ // another collection with a single facet value+ auto coll2 = collectionManager.create_collection("coll2", 4, fields, "rating").get();+ doc["id"] = "0";+ doc["title"] = titles[0];+ doc["rating"] = 8.5;+ coll2->add(doc.dump());++ doc["id"] = "1";+ coll2->add(doc.dump());++ coll2->remove("0");+ results = coll2->search("*", {"title"}, "", facets, sort_fields, {0}, 10, 1,+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 2,"", 30UL, 4UL,+ "", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},+ 4294967295UL, true, false, true, "", false, 6000000UL, 4UL,+ 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();++ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("The Shawshank Redemption", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"]);+}