Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Improve facet aggregation performance

Jason Bosco60799111be0c
diff --git a/include/field.h b/include/field.h
index f1a71447..30562a15 100644
--- a/include/field.h
+++ b/include/field.h
@@ -202,7 +202,7 @@ struct facet_count_t {
uint32_t doc_id;
uint32_t array_pos;
- spp::sparse_hash_map<uint32_t, token_pos_cost_t> query_token_pos;
+ std::unordered_map<uint32_t, token_pos_cost_t> query_token_pos;
};
struct facet_stats_t {
@@ -214,7 +214,7 @@ struct facet_stats_t {
struct facet {
const std::string field_name;
- std::map<uint64_t, facet_count_t> result_map;
+ std::unordered_map<uint64_t, facet_count_t> result_map;
facet_stats_t stats;
facet(const std::string & field_name): field_name(field_name) {
diff --git a/src/index.cpp b/src/index.cpp
index bd61ac52..2ec98834 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -647,7 +647,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
struct facet_info_t {
// facet hash => token position in the query
- spp::sparse_hash_map<uint64_t, token_pos_cost_t> fhash_qtoken_pos;
+ std::unordered_map<uint64_t, token_pos_cost_t> fhash_qtoken_pos;
bool use_facet_query = false;
bool should_compute_stats = false;
@@ -743,7 +743,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
bool fvalue_found = false;
uint64_t combined_hash = 1; // for hashing the entire facet value (multiple tokens)
- spp::sparse_hash_map<uint32_t, token_pos_cost_t> query_token_positions;
+ std::unordered_map<uint32_t, token_pos_cost_t> query_token_positions;
size_t field_token_index = -1;
for(size_t j = 0; j < fhashes.size(); j++) {
@@ -760,13 +760,15 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
compute_facet_stats(a_facet, ftoken_hash, facet_field.type);
}
+ const auto fhash_qtoken_pos_it = fhash_qtoken_pos.find(ftoken_hash);
+
// not using facet query or this particular facet value is found in facet filter
- if(!use_facet_query || fhash_qtoken_pos.find(ftoken_hash) != fhash_qtoken_pos.end()) {
+ if(!use_facet_query || fhash_qtoken_pos_it != fhash_qtoken_pos.end()) {
fvalue_found = true;
if(use_facet_query) {
// map token index to query index (used for highlighting later on)
- token_pos_cost_t qtoken_pos = fhash_qtoken_pos.at(ftoken_hash);
+ const token_pos_cost_t& qtoken_pos = fhash_qtoken_pos_it->second;
// if the query token has already matched another token in the string
// we will replace the position only if the cost is lower
@@ -785,29 +787,31 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
uint64_t fhash = combined_hash;
if(a_facet.result_map.count(fhash) == 0) {
- a_facet.result_map[fhash] = facet_count_t{0, spp::sparse_hash_set<uint64_t>(),
- doc_seq_id, 0,
- spp::sparse_hash_map<uint32_t, token_pos_cost_t>()};
+ a_facet.result_map.emplace(fhash, facet_count_t{0, spp::sparse_hash_set<uint64_t>(),
+ doc_seq_id, 0,
+ std::unordered_map<uint32_t, token_pos_cost_t>()});
}
- a_facet.result_map[fhash].doc_id = doc_seq_id;
- a_facet.result_map[fhash].array_pos = array_pos;
+ facet_count_t& facet_count = a_facet.result_map[fhash];
+
+ facet_count.doc_id = doc_seq_id;
+ facet_count.array_pos = array_pos;
if(search_params->group_limit) {
- a_facet.result_map[fhash].groups.emplace(distinct_id);
+ facet_count.groups.emplace(distinct_id);
} else {
- a_facet.result_map[fhash].count += 1;
+ facet_count.count += 1;
}
if(use_facet_query) {
- a_facet.result_map[fhash].query_token_pos = query_token_positions;
+ facet_count.query_token_pos = query_token_positions;
}
}
array_pos++;
fvalue_found = false;
combined_hash = 1;
- spp::sparse_hash_map<uint32_t, token_pos_cost_t>().swap(query_token_positions);
+ std::unordered_map<uint32_t, token_pos_cost_t>().swap(query_token_positions);
field_token_index = -1;
}
}
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index 4e545b18..e192a2c2 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -283,8 +283,8 @@ TEST_F(CollectionFacetingTest, FacetCounts) {
ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
- ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
- ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
// facet with wildcard query
facets.clear();
@@ -302,8 +302,8 @@ TEST_F(CollectionFacetingTest, FacetCounts) {
ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
- ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
- ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
// facet with facet filter query (allows typo correction!)
results = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,
@@ -586,12 +586,12 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {
ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
ASSERT_EQ(1, results["facet_counts"][0]["counts"][1]["count"].get<size_t>());
- ASSERT_STREQ("Cell Phone Cases & Clips", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>Cell</mark> Phone Cases & Clips", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
ASSERT_EQ(1, results["facet_counts"][0]["counts"][2]["count"].get<size_t>());
- ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("Cell Phone Cases & Clips", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Cell</mark> Phone Cases & Clips", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());
coll1->remove("100");
@@ -610,12 +610,12 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {
ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("All Unlocked Cell Phones", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
- ASSERT_STREQ("All Unlocked <mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("Unlocked Cell Phones", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Unlocked <mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][2]["count"]);
- ASSERT_STREQ("Unlocked Cell Phones", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
- ASSERT_STREQ("Unlocked <mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("All Unlocked Cell Phones", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("All Unlocked <mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());
coll1->remove("100");
doc["categories"] = {"Cell Phones", "Cell Phone Accessories", "Cell Phone Cases & Clips"};
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index b6e1df35..6f35b2da 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -91,10 +91,10 @@ TEST_F(CollectionGroupingTest, GroupingBasics) {
ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
@@ -179,10 +179,10 @@ TEST_F(CollectionGroupingTest, GroupingCompoundKey) {
ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
@@ -215,10 +215,10 @@ TEST_F(CollectionGroupingTest, GroupingCompoundKey) {
ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
@@ -417,13 +417,13 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {
ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);
ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index 40c45c39..37120de4 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -327,10 +327,10 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeFacetFilterQuery) {
ASSERT_EQ(9, results["found"].get<size_t>());
// "count" would be `2` without exclusion
- ASSERT_EQ("Kristin <mark>Scott</mark> Thomas", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>());
+ ASSERT_EQ("<mark>Scott</mark> Glenn", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>());
ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());
- ASSERT_EQ("<mark>Scott</mark> Glenn", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>());
+ ASSERT_EQ("Kristin <mark>Scott</mark> Thomas", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>());
ASSERT_EQ(1, results["facet_counts"][0]["counts"][1]["count"].get<size_t>());
// ensure per_page is respected