Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Move facet tokens and groups to separate maps.

Kishore Nallan774f3d3ce35d
diff --git a/include/field.h b/include/field.h
index 6b4c7418..690148f9 100644
--- a/include/field.h
+++ b/include/field.h
@@ -539,19 +539,11 @@ public:
}
};
-struct token_pos_cost_t {
- size_t pos;
- uint32_t cost;
-};
-
struct facet_count_t {
uint32_t count = 0;
- spp::sparse_hash_set<uint64_t> groups; // used for faceting grouped results
-
// used to fetch the actual document and value for representation
uint32_t doc_id = 0;
uint32_t array_pos = 0;
- std::vector<std::string> tokens;
};
struct facet_stats_t {
@@ -563,10 +555,17 @@ struct facet_stats_t {
struct facet {
const std::string field_name;
- std::unordered_map<uint64_t, facet_count_t> result_map;
+ spp::sparse_hash_map<uint64_t, facet_count_t> result_map;
+
+ // used for facet value query
+ spp::sparse_hash_map<uint64_t, std::vector<std::string>> hash_tokens;
+
+ // used for faceting grouped results
+ spp::sparse_hash_map<uint64_t, spp::sparse_hash_set<uint64_t>> hash_groups;
+
facet_stats_t stats;
- explicit facet(const std::string & field_name): field_name(field_name) {
+ explicit facet(const std::string& field_name): field_name(field_name) {
}
};
diff --git a/src/collection.cpp b/src/collection.cpp
index 1dd85f3a..f875bb68 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -952,7 +952,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
if(group_limit) {
for(auto& acc_facet: facets) {
for(auto& facet_kv: acc_facet.result_map) {
- facet_kv.second.count = facet_kv.second.groups.size();
+ facet_kv.second.count = acc_facet.hash_groups[facet_kv.first].size();
}
}
@@ -1168,7 +1168,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
result["facet_counts"] = nlohmann::json::array();
// populate facets
- for(const facet & a_facet: facets) {
+ for(facet & a_facet: facets) {
nlohmann::json facet_result = nlohmann::json::object();
facet_result["field_name"] = a_facet.field_name;
facet_result["counts"] = nlohmann::json::array();
@@ -1218,21 +1218,22 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
}
std::unordered_map<std::string, size_t> ftoken_pos;
+ std::vector<string>& ftokens = a_facet.hash_tokens[kv.first];
- for(size_t ti = 0; ti < facet_count.tokens.size(); ti++) {
+ for(size_t ti = 0; ti < ftokens.size(); ti++) {
if(the_field.is_bool()) {
- if(facet_count.tokens[ti] == "1") {
- facet_count.tokens[ti] = "true";
+ if(ftokens[ti] == "1") {
+ ftokens[ti] = "true";
} else {
- facet_count.tokens[ti] = "false";
+ ftokens[ti] = "false";
}
}
- const std::string& resolved_token = facet_count.tokens[ti];
+ const std::string& resolved_token = ftokens[ti];
ftoken_pos[resolved_token] = ti;
}
- const std::string& last_full_q_token = facet_count.tokens.empty() ? "" : facet_count.tokens.back();
+ const std::string& last_full_q_token = ftokens.empty() ? "" : ftokens.back();
const std::string& last_q_token = facet_query_tokens.empty() ? "" : facet_query_tokens.back();
// 2 passes: first identify tokens that need to be highlighted and then construct highlighted text
diff --git a/src/index.cpp b/src/index.cpp
index c0688ab0..d77bf416 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1028,13 +1028,13 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
facet_count.array_pos = j;
if(group_limit) {
- facet_count.groups.emplace(distinct_id);
+ a_facet.hash_groups[fhash].emplace(distinct_id);
} else {
facet_count.count += 1;
}
if(use_facet_query) {
- facet_count.tokens = fquery_hashes.at(fhash);
+ a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);
}
}
}
@@ -2415,7 +2415,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,
std::unique_lock<std::mutex> lock_process(m_process);
cv_process.wait(lock_process, [&](){ return num_processed == num_queued; });
- for(const auto& facet_batch: facet_batches) {
+ for(auto& facet_batch: facet_batches) {
for(size_t fi = 0; fi < facet_batch.size(); fi++) {
auto& this_facet = facet_batch[fi];
auto& acc_facet = facets[fi];
@@ -2423,8 +2423,9 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,
for(auto & facet_kv: this_facet.result_map) {
if(group_limit) {
// we have to add all group sets
- acc_facet.result_map[facet_kv.first].groups.insert(
- facet_kv.second.groups.begin(), facet_kv.second.groups.end()
+ acc_facet.hash_groups[facet_kv.first].insert(
+ this_facet.hash_groups[facet_kv.first].begin(),
+ this_facet.hash_groups[facet_kv.first].end()
);
} else {
size_t count = 0;
@@ -2439,7 +2440,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,
acc_facet.result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;
acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;
- acc_facet.result_map[facet_kv.first].tokens = facet_kv.second.tokens;
+ acc_facet.hash_tokens[facet_kv.first] = this_facet.hash_tokens[facet_kv.first];
}
if(this_facet.stats.fvcount != 0) {
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index bbd00c74..c0b63ea7 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -575,8 +575,8 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {
ASSERT_EQ(1, results["facet_counts"].size());
ASSERT_EQ(3, results["facet_counts"][0]["counts"].size());
- ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>Cello</mark>phanes", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Cello</mark>phanes", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index 32e0de7a..09111c88 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -418,10 +418,10 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {
ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 52a73c60..63ceba4f 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -2272,10 +2272,10 @@ TEST_F(CollectionTest, UpdateDocument) {
ASSERT_STREQ("tags", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(2, res["facet_counts"][0]["counts"].size());
- ASSERT_STREQ("NEWS", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("LAZY", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("LAZY", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("NEWS", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][1]["count"]);
// upsert only part of the document -- document should be REPLACED