Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Speed up facet computation for array fields.
kishorenc0004b5d9e28b
diff --git a/include/index.h b/include/index.hindex 0bfd1413..ce2995f0 100644--- a/include/index.h+++ b/include/index.h@@ -190,7 +190,7 @@ private:uint64_t facet_token_hash(const field & a_field, const std::string &token);- void compute_facet_stats(facet &a_facet, int64_t raw_value, const std::string & field_type);+ void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);// reference: https://stackoverflow.com/a/27952689/131050uint64_t hash_combine(uint64_t lhs, uint64_t rhs) const {diff --git a/src/index.cpp b/src/index.cppindex 95389432..5c0e7745 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -587,7 +587,7 @@ void Index::index_float_array_field(const std::vector<float> & values, const uin}}-void Index::compute_facet_stats(facet &a_facet, int64_t raw_value, const std::string & field_type) {+void Index::compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type) {if(field_type == field_types::INT32 || field_type == field_types::INT32_ARRAY) {int32_t val = raw_value;if (val < a_facet.stats.fvmin) {@@ -685,6 +685,11 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,size_t facet_id = facet_to_index[a_facet.field_name];+ bool should_compute_stats = (facet_field.type != field_types::STRING &&+ facet_field.type != field_types::BOOL &&+ facet_field.type != field_types::STRING_ARRAY &&+ facet_field.type != field_types::BOOL_ARRAY);+for(size_t i = 0; i < results_size; i++) {uint32_t doc_seq_id = result_ids[i];@@ -696,18 +701,23 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,int array_pos = 0;bool fvalue_found = false;- std::stringstream fvaluestream; // for hashing the entire facet value (multiple tokens)+ uint64_t combined_hash = 1; // for hashing the entire facet value (multiple tokens)+spp::sparse_hash_map<uint32_t, token_pos_cost_t> query_token_positions;size_t field_token_index = -1;for(size_t j = 0; j < fhashes.size(); j++) {if(fhashes[j] != FACET_ARRAY_DELIMETER) {- int64_t ftoken_hash = fhashes[j];- fvaluestream << ftoken_hash;+ uint64_t ftoken_hash = fhashes[j];++ // reference: https://stackoverflow.com/a/4182771/131050+ combined_hash *= (1779033703 + 2*ftoken_hash);field_token_index++;// ftoken_hash is the raw value for numeric fields- compute_facet_stats(a_facet, ftoken_hash, facet_field.type);+ if(should_compute_stats) {+ compute_facet_stats(a_facet, ftoken_hash, facet_field.type);+ }// not using facet query or this particular facet value is found in facet filterif(!use_facet_query || fhash_qtoken_pos.find(ftoken_hash) != fhash_qtoken_pos.end()) {@@ -731,8 +741,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,// 0 indicates separator, while the second condition checks for non-array stringif(fhashes[j] == FACET_ARRAY_DELIMETER || (fhashes.back() != FACET_ARRAY_DELIMETER && j == fhashes.size() - 1)) {if(!use_facet_query || fvalue_found) {- const std::string & fvalue_str = fvaluestream.str();- uint64_t fhash = facet_token_hash(facet_field, fvalue_str);+ uint64_t fhash = combined_hash;if(a_facet.result_map.count(fhash) == 0) {a_facet.result_map[fhash] = facet_count_t{0, spp::sparse_hash_set<uint64_t>(),@@ -757,7 +766,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,array_pos++;fvalue_found = false;- std::stringstream().swap(fvaluestream);+ combined_hash = 1;spp::sparse_hash_map<uint32_t, token_pos_cost_t>().swap(query_token_positions);field_token_index = -1;}diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex 7d430a7c..49752473 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -137,8 +137,8 @@ TEST_F(CollectionFacetingTest, FacetCounts) {ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());- ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());- ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());+ ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());+ ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());// facet with wildcard queryfacets.clear();@@ -156,8 +156,8 @@ TEST_F(CollectionFacetingTest, FacetCounts) {ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());- ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());- ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());+ ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());+ ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());// facet with facet filter query (allows typo correction!)results = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,@@ -410,15 +410,16 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {ASSERT_EQ(3, results["facet_counts"][0]["counts"].size());ASSERT_STREQ("categories", results["facet_counts"][0]["field_name"].get<std::string>().c_str());- ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("Cell Phones", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());- ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());+ ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());+ ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());- ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][2]["count"]);+ ASSERT_EQ(1, results["facet_counts"][0]["counts"][1]["count"].get<size_t>());+ ASSERT_STREQ("Cell Phones", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());++ ASSERT_EQ(1, results["facet_counts"][0]["counts"][2]["count"].get<size_t>());ASSERT_STREQ("Cell Phone Cases & Clips", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());ASSERT_STREQ("<mark>Cell</mark> Phone Cases & Clips", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());@@ -433,13 +434,14 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {ASSERT_EQ(1, results["facet_counts"].size());ASSERT_STREQ("categories", results["facet_counts"][0]["field_name"].get<std::string>().c_str());+ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("Cell Phones", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("All Unlocked Cell Phones", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("All Unlocked <mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("All Unlocked Cell Phones", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());- ASSERT_STREQ("All Unlocked <mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("Cell Phones", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);ASSERT_STREQ("Unlocked Cell Phones", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());@@ -487,8 +489,8 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {ASSERT_EQ(1, results["facet_counts"].size());ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());- ASSERT_STREQ("Cell Phones", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());- ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Cell Phones", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());collectionManager.drop_collection("coll1");}diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex 07cfa639..9b06dd72 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -89,11 +89,12 @@ TEST_F(CollectionGroupingTest, GroupingBasics) {ASSERT_STREQ("8", res["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());+ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());@@ -176,11 +177,12 @@ TEST_F(CollectionGroupingTest, GroupingCompoundKey) {ASSERT_STREQ("0", res["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());+ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());@@ -211,11 +213,12 @@ TEST_F(CollectionGroupingTest, GroupingCompoundKey) {ASSERT_STREQ("Omega", res["grouped_hits"][0]["group_key"][1].get<std::string>().c_str());ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());+ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());@@ -343,14 +346,15 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {// assert facet countsASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());+ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][1]["count"]);ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());