Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Speed up facet computation for array fields.

kishorenc0004b5d9e28b
diff --git a/include/index.h b/include/index.h
index 0bfd1413..ce2995f0 100644
--- a/include/index.h
+++ b/include/index.h
@@ -190,7 +190,7 @@ private:
uint64_t facet_token_hash(const field & a_field, const std::string &token);
- void compute_facet_stats(facet &a_facet, int64_t raw_value, const std::string & field_type);
+ void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);
// reference: https://stackoverflow.com/a/27952689/131050
uint64_t hash_combine(uint64_t lhs, uint64_t rhs) const {
diff --git a/src/index.cpp b/src/index.cpp
index 95389432..5c0e7745 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -587,7 +587,7 @@ void Index::index_float_array_field(const std::vector<float> & values, const uin
}
}
-void Index::compute_facet_stats(facet &a_facet, int64_t raw_value, const std::string & field_type) {
+void Index::compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type) {
if(field_type == field_types::INT32 || field_type == field_types::INT32_ARRAY) {
int32_t val = raw_value;
if (val < a_facet.stats.fvmin) {
@@ -685,6 +685,11 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
size_t facet_id = facet_to_index[a_facet.field_name];
+ bool should_compute_stats = (facet_field.type != field_types::STRING &&
+ facet_field.type != field_types::BOOL &&
+ facet_field.type != field_types::STRING_ARRAY &&
+ facet_field.type != field_types::BOOL_ARRAY);
+
for(size_t i = 0; i < results_size; i++) {
uint32_t doc_seq_id = result_ids[i];
@@ -696,18 +701,23 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
int array_pos = 0;
bool fvalue_found = false;
- std::stringstream fvaluestream; // for hashing the entire facet value (multiple tokens)
+ uint64_t combined_hash = 1; // for hashing the entire facet value (multiple tokens)
+
spp::sparse_hash_map<uint32_t, token_pos_cost_t> query_token_positions;
size_t field_token_index = -1;
for(size_t j = 0; j < fhashes.size(); j++) {
if(fhashes[j] != FACET_ARRAY_DELIMETER) {
- int64_t ftoken_hash = fhashes[j];
- fvaluestream << ftoken_hash;
+ uint64_t ftoken_hash = fhashes[j];
+
+ // reference: https://stackoverflow.com/a/4182771/131050
+ combined_hash *= (1779033703 + 2*ftoken_hash);
field_token_index++;
// ftoken_hash is the raw value for numeric fields
- compute_facet_stats(a_facet, ftoken_hash, facet_field.type);
+ if(should_compute_stats) {
+ compute_facet_stats(a_facet, ftoken_hash, facet_field.type);
+ }
// not using facet query or this particular facet value is found in facet filter
if(!use_facet_query || fhash_qtoken_pos.find(ftoken_hash) != fhash_qtoken_pos.end()) {
@@ -731,8 +741,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
// 0 indicates separator, while the second condition checks for non-array string
if(fhashes[j] == FACET_ARRAY_DELIMETER || (fhashes.back() != FACET_ARRAY_DELIMETER && j == fhashes.size() - 1)) {
if(!use_facet_query || fvalue_found) {
- const std::string & fvalue_str = fvaluestream.str();
- uint64_t fhash = facet_token_hash(facet_field, fvalue_str);
+ uint64_t fhash = combined_hash;
if(a_facet.result_map.count(fhash) == 0) {
a_facet.result_map[fhash] = facet_count_t{0, spp::sparse_hash_set<uint64_t>(),
@@ -757,7 +766,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
array_pos++;
fvalue_found = false;
- std::stringstream().swap(fvaluestream);
+ combined_hash = 1;
spp::sparse_hash_map<uint32_t, token_pos_cost_t>().swap(query_token_positions);
field_token_index = -1;
}
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index 7d430a7c..49752473 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -137,8 +137,8 @@ TEST_F(CollectionFacetingTest, FacetCounts) {
ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
- ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
- ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
// facet with wildcard query
facets.clear();
@@ -156,8 +156,8 @@ TEST_F(CollectionFacetingTest, FacetCounts) {
ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
- ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
- ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
// facet with facet filter query (allows typo correction!)
results = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,
@@ -410,15 +410,16 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {
ASSERT_EQ(3, results["facet_counts"][0]["counts"].size());
ASSERT_STREQ("categories", results["facet_counts"][0]["field_name"].get<std::string>().c_str());
- ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("Cell Phones", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
- ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());
+ ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
- ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][2]["count"]);
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"][1]["count"].get<size_t>());
+ ASSERT_STREQ("Cell Phones", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
+
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"][2]["count"].get<size_t>());
ASSERT_STREQ("Cell Phone Cases & Clips", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
ASSERT_STREQ("<mark>Cell</mark> Phone Cases & Clips", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());
@@ -433,13 +434,14 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {
ASSERT_EQ(1, results["facet_counts"].size());
ASSERT_STREQ("categories", results["facet_counts"][0]["field_name"].get<std::string>().c_str());
+
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("Cell Phones", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("All Unlocked Cell Phones", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("All Unlocked <mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("All Unlocked Cell Phones", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
- ASSERT_STREQ("All Unlocked <mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("Cell Phones", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);
ASSERT_STREQ("Unlocked Cell Phones", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
@@ -487,8 +489,8 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {
ASSERT_EQ(1, results["facet_counts"].size());
ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
- ASSERT_STREQ("Cell Phones", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
- ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Cell Phones", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index 07cfa639..9b06dd72 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -89,11 +89,12 @@ TEST_F(CollectionGroupingTest, GroupingBasics) {
ASSERT_STREQ("8", res["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
+
ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
@@ -176,11 +177,12 @@ TEST_F(CollectionGroupingTest, GroupingCompoundKey) {
ASSERT_STREQ("0", res["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
+
ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
@@ -211,11 +213,12 @@ TEST_F(CollectionGroupingTest, GroupingCompoundKey) {
ASSERT_STREQ("Omega", res["grouped_hits"][0]["group_key"][1].get<std::string>().c_str());
ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
+
ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
@@ -343,14 +346,15 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {
// assert facet counts
ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
+
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][1]["count"]);
ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);
ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());