Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix facet counter list reordering.

Kishore Nallanc0a507750ad8
diff --git a/include/facet_index.h b/include/facet_index.h
index fb3ce3b7..998471c0 100644
--- a/include/facet_index.h
+++ b/include/facet_index.h
@@ -80,6 +80,7 @@ private:
struct facet_doc_ids_list_t {
std::map<std::string, facet_id_seq_ids_t> fvalue_seq_ids;
std::list<facet_count_t> counts;
+ std::map<uint32_t, std::list<facet_count_t>::iterator> count_map;
posting_list_t* seq_id_hashes = nullptr;
spp::sparse_hash_map<uint32_t, int64_t> fhash_to_int64_map;
@@ -154,4 +155,9 @@ public:
//get fhash=>int64 map for stats
const spp::sparse_hash_map<uint32_t, int64_t>& get_fhash_int64_map(const std::string& field_name);
+
+ void update_count_nodes(std::list<facet_count_t>& count_list,
+ std::map<uint32_t, std::list<facet_count_t>::iterator>& count_map,
+ uint32_t old_count, uint32_t new_count,
+ std::list<facet_count_t>::iterator& curr) const;
};
\ No newline at end of file
diff --git a/src/facet_index.cpp b/src/facet_index.cpp
index 3ac86418..119f6e7c 100644
--- a/src/facet_index.cpp
+++ b/src/facet_index.cpp
@@ -2,6 +2,7 @@
#include <tokenizer.h>
#include "string_utils.h"
#include "array_utils.h"
+#include "bazel-src/external/onnx_runtime/cmake/external/protobuf/src/google/protobuf/map.h"
void facet_index_t::initialize(const std::string& field) {
const auto facet_field_map_it = facet_field_map.find(field);
@@ -59,9 +60,31 @@ void facet_index_t::insert(const std::string& field_name,std::unordered_map<face
fis.facet_id = facet_id;
if(facet_index.has_value_index) {
- count_list.emplace_back(fvalue.facet_value, seq_ids.size(), facet_id);
- fis.facet_count_it = std::prev(count_list.end());
+ auto new_count = seq_ids.size();
fis.seq_ids = ids_t::create(seq_ids);
+
+ auto& count_map = facet_index.count_map;
+ auto count_map_it = count_map.lower_bound(new_count);
+
+ count_list.emplace_front(fvalue.facet_value, new_count, facet_id);
+ fis.facet_count_it = count_list.begin();
+ auto curr = fis.facet_count_it;
+
+ if(count_map_it == facet_index.count_map.end()) {
+ // no element >= new_count, so it remains at head of list
+ facet_index.count_map.emplace(curr->count, curr);
+ } else if(count_map_it->second->count == new_count) {
+ // `curr` must be placed _before_ anchor node
+ // a) [7], 10, <7>, 3
+ auto& existing_node = count_map_it->second;
+ count_list.splice(existing_node, count_list, curr);
+ } else {
+ // b) [9], 12, 10, <10>, 7, 3
+ // `curr` placed _after_ anchor node
+ auto& gt_node = count_map_it->second;
+ count_list.splice(std::next(gt_node), count_list, curr);
+ count_map.emplace(curr->count, curr);
+ }
}
fvalue_index.emplace(fvalue.facet_value, fis);
@@ -73,12 +96,14 @@ void facet_index_t::insert(const std::string& field_name,std::unordered_map<face
auto facet_count_it = fvalue_index_it->second.facet_count_it;
if(facet_count_it->facet_id == facet_id) {
+ auto& count_map = facet_index.count_map;
+ auto old_count = facet_count_it->count;
facet_count_it->count = ids_t::num_ids(fvalue_index_it->second.seq_ids);
+ auto new_count = facet_count_it->count;
+
+ // Relocate `curr` to a new sorted location within count list:
auto curr = facet_count_it;
- while (curr != count_list.begin() && std::prev(curr)->count < curr->count) {
- count_list.splice(curr, count_list, std::prev(curr)); // swaps list nodes
- curr--;
- }
+ update_count_nodes(count_list, count_map, old_count, new_count, curr);
} else {
LOG(ERROR) << "Wrong reference stored for facet " << fvalue.facet_value << " with facet_id " << facet_id;
}
@@ -93,6 +118,79 @@ void facet_index_t::insert(const std::string& field_name,std::unordered_map<face
}
}
+void facet_index_t::update_count_nodes(std::list<facet_count_t>& count_list,
+ std::map<uint32_t, std::list<facet_count_t>::iterator>& count_map,
+ uint32_t old_count, uint32_t new_count,
+ std::list<facet_count_t>::iterator& curr) const {
+
+ auto count_map_it = count_map.lower_bound(new_count);
+
+ if(count_map_it == count_map.end()) {
+ // all elements are < new_count
+
+ // 5, 4, [4 -> 7]
+ // 5, 4, [4 -> 7], 3
+ // 5, [4 -> 7]
+
+ // delete count map entry if `curr` is the anchor for `old_count`
+ if(std::next(curr) == count_list.end() || std::next(curr)->count < old_count) {
+ count_map.erase(old_count);
+
+ // find a replacement for orig_count
+ if(std::prev(curr)->count == old_count) {
+ count_map.emplace(old_count, std::prev(curr));
+ }
+ }
+
+ // `curr` placed at head of the count list
+ count_list.splice(count_list.begin(), count_list, curr);
+ count_map.emplace(new_count, curr);
+ } else if(count_map_it->first == new_count) {
+ // entry for new_count already exists in count map
+ // a) 10, 7, [5 -> 7], 3
+ // 10, 7, 5, [5 -> 7]
+ // 10, 7, [5 -> 7]
+ // 10, 7, [5 -> 7], 5
+
+ auto existing_node = count_map_it->second;
+
+ // delete count map entry if `curr` is the anchor for `old_count`
+ if(std::next(curr) == count_list.end() || std::next(curr)->count < old_count) {
+ count_map.erase(old_count);
+
+ // find a replacement for orig_count
+ if(std::prev(curr)->count == old_count) {
+ count_map.emplace(old_count, std::prev(curr));
+ }
+ }
+
+ // `curr` placed before anchor node
+ count_list.splice(existing_node, count_list, curr);
+ } else {
+ // b) 10, 7, [7 -> 9], 3
+ // 10, 7, [7 -> 9]
+ // 10, [7 -> 9]
+ // 10, [7 -> 9], 7
+ // 10, 7, [5 -> 9], 3
+
+ auto gt_node = count_map_it->second;
+
+ // delete old entry if `orig_count` iterator is same as `curr`
+ if(std::next(curr) == count_list.end() || std::next(curr)->count < old_count) {
+ count_map.erase(old_count);
+
+ // find a replacement for orig_count
+ if(std::prev(curr)->count == old_count) {
+ count_map.emplace(old_count, std::prev(curr));
+ }
+ }
+
+ // `curr` placed after anchor node
+ count_list.splice(std::next(gt_node), count_list, curr);
+ count_map.emplace(new_count, curr);
+ }
+}
+
bool facet_index_t::contains(const std::string& field_name) {
const auto& facet_field_it = facet_field_map.find(field_name);
if(facet_field_it == facet_field_map.end()) {
@@ -117,7 +215,14 @@ void facet_index_t::remove(const std::string& field_name, const uint32_t seq_id)
if(ids && ids_t::contains(ids, seq_id)) {
ids_t::erase(ids, seq_id);
auto& count_list = facet_field_it->second.counts;
- facet_ids_seq_ids->second.facet_count_it->count--;
+ auto curr = facet_ids_seq_ids->second.facet_count_it;
+ auto old_count = curr->count;
+ curr->count--;
+ auto new_count = curr->count;
+
+ // move the node lower in the count list
+ auto& count_map = facet_field_it->second.count_map;
+ update_count_nodes(count_list, count_map, old_count, new_count, curr);
if(ids_t::num_ids(ids) == 0) {
ids_t::destroy_list(ids);
@@ -129,7 +234,9 @@ void facet_index_t::remove(const std::string& field_name, const uint32_t seq_id)
uint32_t fhash = facet_ids_seq_ids->second.facet_id;
fhash_int64_map.erase(fhash);
- count_list.erase(facet_ids_seq_ids->second.facet_count_it);
+ count_map.erase(new_count);
+ count_list.erase(curr);
+
auto node = facet_index_map.extract(facet_ids_seq_ids->first);
node.key() = dead_fvalue;
facet_index_map.insert(std::move(node));
diff --git a/test/collection_optimized_faceting_test.cpp b/test/collection_optimized_faceting_test.cpp
index 9a9f76fa..95a80eec 100644
--- a/test/collection_optimized_faceting_test.cpp
+++ b/test/collection_optimized_faceting_test.cpp
@@ -2195,14 +2195,144 @@ TEST_F(CollectionOptimizedFacetingTest, ValueIndexStatsMinMax) {
ASSERT_EQ(1, results["facet_counts"].size());
ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
- ASSERT_EQ("9.3", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("9", results["facet_counts"][0]["counts"][0]["value"]);
ASSERT_EQ("9.2", results["facet_counts"][0]["counts"][1]["value"]);
//stats
ASSERT_EQ(5, results["facet_counts"][0]["stats"].size());
- ASSERT_FLOAT_EQ(9.25, results["facet_counts"][0]["stats"]["avg"].get<double>());
+ ASSERT_FLOAT_EQ(9.1, results["facet_counts"][0]["stats"]["avg"].get<double>());
ASSERT_FLOAT_EQ(8.800000190734863, results["facet_counts"][0]["stats"]["min"].get<double>());
ASSERT_FLOAT_EQ(9.300000190734863, results["facet_counts"][0]["stats"]["max"].get<double>());
- ASSERT_FLOAT_EQ(18.5, results["facet_counts"][0]["stats"]["sum"].get<double>());
+ ASSERT_FLOAT_EQ(18.2, results["facet_counts"][0]["stats"]["sum"].get<double>());
ASSERT_FLOAT_EQ(2, results["facet_counts"][0]["stats"]["total_values"].get<size_t>());
-}
\ No newline at end of file
+}
+
+TEST_F(CollectionOptimizedFacetingTest, StringFacetsCountListOrderTest) {
+ //check if count list is ordering facets
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, true),
+ field("rating", field_types::FLOAT, false)};
+
+ std::vector<sort_by> sort_fields = {sort_by("rating", "DESC")};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "rating").get();
+ }
+
+ std::vector<std::string> titles {"The Shawshank Redemption", "The Godfather", "The Dark Knight"};
+ nlohmann::json doc;
+ int i = 0;
+ for(; i < 6; ++i) {
+ doc["id"] = std::to_string(i);
+ doc["title"] = titles[i%3];
+ doc["rating"] = 8.5;
+ coll1->add(doc.dump());
+ }
+
+ //add last title more
+ for(; i < 10; ++i) {
+ doc["id"] = std::to_string(i);
+ doc["title"] = titles[2];
+ doc["rating"] = 8.5;
+ coll1->add(doc.dump());
+ }
+
+ std::vector<std::string> facets = {"title"};
+
+ //limit max facets to 2
+ nlohmann::json results = coll1->search("*", {"title"}, "", facets, sort_fields, {0}, 10, 1,
+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 2,"", 30UL, 4UL,
+ "", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},
+ 4294967295UL, true, false, true, "", false, 6000000UL, 4UL,
+ 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,
+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();
+
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("The Dark Knight", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ(6, results["facet_counts"][0]["counts"][0]["count"]);
+ ASSERT_EQ("The Godfather", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"][1]["count"]);
+}
+
+TEST_F(CollectionOptimizedFacetingTest, StringFacetsCountListRemoveTest) {
+ //delete records and check if counts are updated
+
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, true),
+ field("rating", field_types::FLOAT, false)};
+
+ std::vector<sort_by> sort_fields = {sort_by("rating", "DESC")};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "rating").get();
+ }
+
+ std::vector<std::string> titles {"The Shawshank Redemption", "The Godfather", "The Dark Knight"};
+ nlohmann::json doc;
+ int i = 0;
+ for(; i < 6; ++i) {
+ doc["id"] = std::to_string(i);
+ doc["title"] = titles[i%3];
+ doc["rating"] = 8.5;
+ coll1->add(doc.dump());
+ }
+
+ //add last title more
+ for(; i < 10; ++i) {
+ doc["id"] = std::to_string(i);
+ doc["title"] = titles[2];
+ doc["rating"] = 8.5;
+ coll1->add(doc.dump());
+ }
+
+ // remove first doc
+ coll1->remove("0");
+
+ std::vector<std::string> facets = {"title"};
+
+ // limit max facets to 2
+ nlohmann::json results = coll1->search("*", {"title"}, "", facets, sort_fields, {0}, 10, 1,
+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 2,"", 30UL, 4UL,
+ "", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},
+ 4294967295UL, true, false, true, "", false, 6000000UL, 4UL,
+ 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,
+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();
+
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("The Dark Knight", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ(6, results["facet_counts"][0]["counts"][0]["count"]);
+ ASSERT_EQ("The Godfather", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"][1]["count"]);
+
+ // another collection with a single facet value
+ auto coll2 = collectionManager.create_collection("coll2", 4, fields, "rating").get();
+ doc["id"] = "0";
+ doc["title"] = titles[0];
+ doc["rating"] = 8.5;
+ coll2->add(doc.dump());
+
+ doc["id"] = "1";
+ coll2->add(doc.dump());
+
+ coll2->remove("0");
+ results = coll2->search("*", {"title"}, "", facets, sort_fields, {0}, 10, 1,
+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 2,"", 30UL, 4UL,
+ "", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},
+ 4294967295UL, true, false, true, "", false, 6000000UL, 4UL,
+ 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,
+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();
+
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("The Shawshank Redemption", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"]);
+}