Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Improve facet aggregation performance
Jason Bosco60799111be0c
diff --git a/include/field.h b/include/field.hindex f1a71447..30562a15 100644--- a/include/field.h+++ b/include/field.h@@ -202,7 +202,7 @@ struct facet_count_t {uint32_t doc_id;uint32_t array_pos;- spp::sparse_hash_map<uint32_t, token_pos_cost_t> query_token_pos;+ std::unordered_map<uint32_t, token_pos_cost_t> query_token_pos;};struct facet_stats_t {@@ -214,7 +214,7 @@ struct facet_stats_t {struct facet {const std::string field_name;- std::map<uint64_t, facet_count_t> result_map;+ std::unordered_map<uint64_t, facet_count_t> result_map;facet_stats_t stats;facet(const std::string & field_name): field_name(field_name) {diff --git a/src/index.cpp b/src/index.cppindex bd61ac52..2ec98834 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -647,7 +647,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,struct facet_info_t {// facet hash => token position in the query- spp::sparse_hash_map<uint64_t, token_pos_cost_t> fhash_qtoken_pos;+ std::unordered_map<uint64_t, token_pos_cost_t> fhash_qtoken_pos;bool use_facet_query = false;bool should_compute_stats = false;@@ -743,7 +743,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,bool fvalue_found = false;uint64_t combined_hash = 1; // for hashing the entire facet value (multiple tokens)- spp::sparse_hash_map<uint32_t, token_pos_cost_t> query_token_positions;+ std::unordered_map<uint32_t, token_pos_cost_t> query_token_positions;size_t field_token_index = -1;for(size_t j = 0; j < fhashes.size(); j++) {@@ -760,13 +760,15 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,compute_facet_stats(a_facet, ftoken_hash, facet_field.type);}+ const auto fhash_qtoken_pos_it = fhash_qtoken_pos.find(ftoken_hash);+// not using facet query or this particular facet value is found in facet filter- if(!use_facet_query || fhash_qtoken_pos.find(ftoken_hash) != fhash_qtoken_pos.end()) {+ if(!use_facet_query || fhash_qtoken_pos_it != fhash_qtoken_pos.end()) {fvalue_found = true;if(use_facet_query) {// map token index to query index (used for highlighting later on)- token_pos_cost_t qtoken_pos = fhash_qtoken_pos.at(ftoken_hash);+ const token_pos_cost_t& qtoken_pos = fhash_qtoken_pos_it->second;// if the query token has already matched another token in the string// we will replace the position only if the cost is lower@@ -785,29 +787,31 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,uint64_t fhash = combined_hash;if(a_facet.result_map.count(fhash) == 0) {- a_facet.result_map[fhash] = facet_count_t{0, spp::sparse_hash_set<uint64_t>(),- doc_seq_id, 0,- spp::sparse_hash_map<uint32_t, token_pos_cost_t>()};+ a_facet.result_map.emplace(fhash, facet_count_t{0, spp::sparse_hash_set<uint64_t>(),+ doc_seq_id, 0,+ std::unordered_map<uint32_t, token_pos_cost_t>()});}- a_facet.result_map[fhash].doc_id = doc_seq_id;- a_facet.result_map[fhash].array_pos = array_pos;+ facet_count_t& facet_count = a_facet.result_map[fhash];++ facet_count.doc_id = doc_seq_id;+ facet_count.array_pos = array_pos;if(search_params->group_limit) {- a_facet.result_map[fhash].groups.emplace(distinct_id);+ facet_count.groups.emplace(distinct_id);} else {- a_facet.result_map[fhash].count += 1;+ facet_count.count += 1;}if(use_facet_query) {- a_facet.result_map[fhash].query_token_pos = query_token_positions;+ facet_count.query_token_pos = query_token_positions;}}array_pos++;fvalue_found = false;combined_hash = 1;- spp::sparse_hash_map<uint32_t, token_pos_cost_t>().swap(query_token_positions);+ std::unordered_map<uint32_t, token_pos_cost_t>().swap(query_token_positions);field_token_index = -1;}}diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex 4e545b18..e192a2c2 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -283,8 +283,8 @@ TEST_F(CollectionFacetingTest, FacetCounts) {ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());- ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());- ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());+ ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());+ ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());// facet with wildcard queryfacets.clear();@@ -302,8 +302,8 @@ TEST_F(CollectionFacetingTest, FacetCounts) {ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());- ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());- ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());+ ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());+ ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());// facet with facet filter query (allows typo correction!)results = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,@@ -586,12 +586,12 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());ASSERT_EQ(1, results["facet_counts"][0]["counts"][1]["count"].get<size_t>());- ASSERT_STREQ("Cell Phone Cases & Clips", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Cell</mark> Phone Cases & Clips", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());ASSERT_EQ(1, results["facet_counts"][0]["counts"][2]["count"].get<size_t>());- ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("Cell Phone Cases & Clips", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Cell</mark> Phone Cases & Clips", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());coll1->remove("100");@@ -610,12 +610,12 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("All Unlocked Cell Phones", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());- ASSERT_STREQ("All Unlocked <mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("Unlocked Cell Phones", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Unlocked <mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][2]["count"]);- ASSERT_STREQ("Unlocked Cell Phones", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());- ASSERT_STREQ("Unlocked <mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("All Unlocked Cell Phones", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());+ ASSERT_STREQ("All Unlocked <mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());coll1->remove("100");doc["categories"] = {"Cell Phones", "Cell Phone Accessories", "Cell Phone Cases & Clips"};diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex b6e1df35..6f35b2da 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -91,10 +91,10 @@ TEST_F(CollectionGroupingTest, GroupingBasics) {ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());@@ -179,10 +179,10 @@ TEST_F(CollectionGroupingTest, GroupingCompoundKey) {ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());@@ -215,10 +215,10 @@ TEST_F(CollectionGroupingTest, GroupingCompoundKey) {ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());@@ -417,13 +417,13 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex 40c45c39..37120de4 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -327,10 +327,10 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeFacetFilterQuery) {ASSERT_EQ(9, results["found"].get<size_t>());// "count" would be `2` without exclusion- ASSERT_EQ("Kristin <mark>Scott</mark> Thomas", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>());+ ASSERT_EQ("<mark>Scott</mark> Glenn", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>());ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());- ASSERT_EQ("<mark>Scott</mark> Glenn", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>());+ ASSERT_EQ("Kristin <mark>Scott</mark> Thomas", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>());ASSERT_EQ(1, results["facet_counts"][0]["counts"][1]["count"].get<size_t>());// ensure per_page is respected