Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Move facet tokens and groups to separate maps.
Kishore Nallan774f3d3ce35d
diff --git a/include/field.h b/include/field.hindex 6b4c7418..690148f9 100644--- a/include/field.h+++ b/include/field.h@@ -539,19 +539,11 @@ public:}};-struct token_pos_cost_t {- size_t pos;- uint32_t cost;-};-struct facet_count_t {uint32_t count = 0;- spp::sparse_hash_set<uint64_t> groups; // used for faceting grouped results-// used to fetch the actual document and value for representationuint32_t doc_id = 0;uint32_t array_pos = 0;- std::vector<std::string> tokens;};struct facet_stats_t {@@ -563,10 +555,17 @@ struct facet_stats_t {struct facet {const std::string field_name;- std::unordered_map<uint64_t, facet_count_t> result_map;+ spp::sparse_hash_map<uint64_t, facet_count_t> result_map;++ // used for facet value query+ spp::sparse_hash_map<uint64_t, std::vector<std::string>> hash_tokens;++ // used for faceting grouped results+ spp::sparse_hash_map<uint64_t, spp::sparse_hash_set<uint64_t>> hash_groups;+facet_stats_t stats;- explicit facet(const std::string & field_name): field_name(field_name) {+ explicit facet(const std::string& field_name): field_name(field_name) {}};diff --git a/src/collection.cpp b/src/collection.cppindex 1dd85f3a..f875bb68 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -952,7 +952,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sif(group_limit) {for(auto& acc_facet: facets) {for(auto& facet_kv: acc_facet.result_map) {- facet_kv.second.count = facet_kv.second.groups.size();+ facet_kv.second.count = acc_facet.hash_groups[facet_kv.first].size();}}@@ -1168,7 +1168,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sresult["facet_counts"] = nlohmann::json::array();// populate facets- for(const facet & a_facet: facets) {+ for(facet & a_facet: facets) {nlohmann::json facet_result = nlohmann::json::object();facet_result["field_name"] = a_facet.field_name;facet_result["counts"] = nlohmann::json::array();@@ -1218,21 +1218,22 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s}std::unordered_map<std::string, size_t> ftoken_pos;+ std::vector<string>& ftokens = a_facet.hash_tokens[kv.first];- for(size_t ti = 0; ti < facet_count.tokens.size(); ti++) {+ for(size_t ti = 0; ti < ftokens.size(); ti++) {if(the_field.is_bool()) {- if(facet_count.tokens[ti] == "1") {- facet_count.tokens[ti] = "true";+ if(ftokens[ti] == "1") {+ ftokens[ti] = "true";} else {- facet_count.tokens[ti] = "false";+ ftokens[ti] = "false";}}- const std::string& resolved_token = facet_count.tokens[ti];+ const std::string& resolved_token = ftokens[ti];ftoken_pos[resolved_token] = ti;}- const std::string& last_full_q_token = facet_count.tokens.empty() ? "" : facet_count.tokens.back();+ const std::string& last_full_q_token = ftokens.empty() ? "" : ftokens.back();const std::string& last_q_token = facet_query_tokens.empty() ? "" : facet_query_tokens.back();// 2 passes: first identify tokens that need to be highlighted and then construct highlighted textdiff --git a/src/index.cpp b/src/index.cppindex c0688ab0..d77bf416 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1028,13 +1028,13 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,facet_count.array_pos = j;if(group_limit) {- facet_count.groups.emplace(distinct_id);+ a_facet.hash_groups[fhash].emplace(distinct_id);} else {facet_count.count += 1;}if(use_facet_query) {- facet_count.tokens = fquery_hashes.at(fhash);+ a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);}}}@@ -2415,7 +2415,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,std::unique_lock<std::mutex> lock_process(m_process);cv_process.wait(lock_process, [&](){ return num_processed == num_queued; });- for(const auto& facet_batch: facet_batches) {+ for(auto& facet_batch: facet_batches) {for(size_t fi = 0; fi < facet_batch.size(); fi++) {auto& this_facet = facet_batch[fi];auto& acc_facet = facets[fi];@@ -2423,8 +2423,9 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,for(auto & facet_kv: this_facet.result_map) {if(group_limit) {// we have to add all group sets- acc_facet.result_map[facet_kv.first].groups.insert(- facet_kv.second.groups.begin(), facet_kv.second.groups.end()+ acc_facet.hash_groups[facet_kv.first].insert(+ this_facet.hash_groups[facet_kv.first].begin(),+ this_facet.hash_groups[facet_kv.first].end());} else {size_t count = 0;@@ -2439,7 +2440,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,acc_facet.result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;- acc_facet.result_map[facet_kv.first].tokens = facet_kv.second.tokens;+ acc_facet.hash_tokens[facet_kv.first] = this_facet.hash_tokens[facet_kv.first];}if(this_facet.stats.fvcount != 0) {diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex bbd00c74..c0b63ea7 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -575,8 +575,8 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {ASSERT_EQ(1, results["facet_counts"].size());ASSERT_EQ(3, results["facet_counts"][0]["counts"].size());- ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Cello</mark>phanes", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Cello</mark>phanes", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());collectionManager.drop_collection("coll1");diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex 32e0de7a..09111c88 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -418,10 +418,10 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 52a73c60..63ceba4f 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -2272,10 +2272,10 @@ TEST_F(CollectionTest, UpdateDocument) {ASSERT_STREQ("tags", res["facet_counts"][0]["field_name"].get<std::string>().c_str());ASSERT_EQ(2, res["facet_counts"][0]["counts"].size());- ASSERT_STREQ("NEWS", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("LAZY", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("LAZY", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("NEWS", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][1]["count"]);// upsert only part of the document -- document should be REPLACED