Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Restore concurrent facet processing.

Kishore Nallan614f3c290c2f
diff --git a/include/field.h b/include/field.h
index 8e89c502..808c1398 100644
--- a/include/field.h
+++ b/include/field.h
@@ -689,6 +689,8 @@ struct facet {
std::string sort_field="";
+ uint32_t orig_index;
+
bool get_range(int64_t key, std::pair<int64_t, std::string>& range_pair) {
if(facet_range_map.empty()) {
LOG (ERROR) << "Facet range is not defined!!!";
@@ -707,10 +709,10 @@ struct facet {
explicit facet(const std::string& field_name, std::map<int64_t, std::string> facet_range = {},
bool is_range_q = false, bool sort_by_alpha=false, const std::string& order="",
- const std::string& sort_by_field="")
+ const std::string& sort_by_field="", uint32_t orig_index = 0)
: field_name(field_name), facet_range_map(facet_range),
is_range_query(is_range_q), is_sort_by_alpha(sort_by_alpha), sort_order(order),
- sort_field(sort_by_field) {
+ sort_field(sort_by_field), orig_index(orig_index) {
}
};
diff --git a/include/index.h b/include/index.h
index b009096d..ab09ff33 100644
--- a/include/index.h
+++ b/include/index.h
@@ -579,7 +579,8 @@ private:
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
Topster* curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries) const;
- static void compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type);
+ static void compute_facet_stats(facet &a_facet, const std::string& raw_value,
+ const std::string & field_type, const size_t count);
static void compute_facet_stats(facet &a_facet, const int64_t raw_value, const std::string & field_type);
@@ -1064,6 +1065,8 @@ public:
friend class filter_result_iterator_t;
void repair_hnsw_index();
+
+ void aggregate_facet(const size_t group_limit, facet& this_facet, facet& acc_facet) const;
};
template<class T>
diff --git a/src/index.cpp b/src/index.cpp
index 56200c0b..86fc0124 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1225,7 +1225,8 @@ void Index::initialize_facet_indexes(const field& facet_field) {
facet_index_v4->initialize(facet_field.name);
}
-void Index::compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type) {
+void Index::compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type,
+ const size_t count) {
if(field_type == field_types::INT32 || field_type == field_types::INT32_ARRAY) {
int32_t val = std::stoi(raw_value);
if (val < a_facet.stats.fvmin) {
@@ -1234,8 +1235,8 @@ void Index::compute_facet_stats(facet &a_facet, const std::string& raw_value, co
if (val > a_facet.stats.fvmax) {
a_facet.stats.fvmax = val;
}
- a_facet.stats.fvsum += val;
- a_facet.stats.fvcount++;
+ a_facet.stats.fvsum += (count * val);
+ a_facet.stats.fvcount += count;
} else if(field_type == field_types::INT64 || field_type == field_types::INT64_ARRAY) {
int64_t val = std::stoll(raw_value);
if(val < a_facet.stats.fvmin) {
@@ -1244,8 +1245,8 @@ void Index::compute_facet_stats(facet &a_facet, const std::string& raw_value, co
if(val > a_facet.stats.fvmax) {
a_facet.stats.fvmax = val;
}
- a_facet.stats.fvsum += val;
- a_facet.stats.fvcount++;
+ a_facet.stats.fvsum += (count * val);
+ a_facet.stats.fvcount += count;
} else if(field_type == field_types::FLOAT || field_type == field_types::FLOAT_ARRAY) {
float val = std::stof(raw_value);
if(val < a_facet.stats.fvmin) {
@@ -1254,8 +1255,8 @@ void Index::compute_facet_stats(facet &a_facet, const std::string& raw_value, co
if(val > a_facet.stats.fvmax) {
a_facet.stats.fvmax = val;
}
- a_facet.stats.fvsum += val;
- a_facet.stats.fvcount++;
+ a_facet.stats.fvsum += (count * val);
+ a_facet.stats.fvcount += count;
}
}
@@ -1396,7 +1397,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
if(should_compute_stats) {
//LOG(INFO) << "Computing facet stats for facet value" << kv.first;
- compute_facet_stats(a_facet, kv.first, facet_field.type);
+ compute_facet_stats(a_facet, kv.first, facet_field.type, kv.second.count);
}
}
@@ -3157,7 +3158,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
bool is_wildcard_no_filter_query = is_wildcard_non_phrase_query && no_filters_provided;
if(!facets.empty()) {
- const size_t num_threads = 1; //std::min(concurrency, all_result_ids_len);
+ const size_t num_threads = std::min(concurrency, all_result_ids_len);
const size_t window_size = (num_threads == 0) ? 0 :
(all_result_ids_len + num_threads - 1) / num_threads; // rounds up
@@ -3177,11 +3178,26 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
max_candidates, facet_infos, facet_index_type);
std::vector<std::vector<facet>> facet_batches(num_threads);
- for(size_t i = 0; i < num_threads; i++) {
- for(const auto& this_facet: facets) {
- facet_batches[i].emplace_back(facet(this_facet.field_name, this_facet.facet_range_map,
- this_facet.is_range_query, this_facet.is_sort_by_alpha, this_facet.sort_order,
- this_facet.sort_field));
+ std::vector<facet> value_facets;
+
+ for(size_t i = 0; i < facets.size(); i++) {
+ const auto& this_facet = facets[i];
+#ifdef TEST_BUILD
+ if(facet_index_type == VALUE) {
+#else
+ if(facet_infos[i].use_value_index) {
+#endif
+ // value based faceting on a single thread
+ value_facets.emplace_back(this_facet.field_name, this_facet.facet_range_map,
+ this_facet.is_range_query, this_facet.is_sort_by_alpha,
+ this_facet.sort_order, this_facet.sort_field, i);
+ continue;
+ }
+
+ for(size_t j = 0; j < num_threads; j++) {
+ facet_batches[j].emplace_back(this_facet.field_name, this_facet.facet_range_map,
+ this_facet.is_range_query, this_facet.is_sort_by_alpha,
+ this_facet.sort_order, this_facet.sort_field, i);
}
}
@@ -3201,6 +3217,10 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
batch_res_len = all_result_ids_len - result_index;
}
+ if(facet_batches[thread_id].empty()) {
+ continue;
+ }
+
uint32_t* batch_result_ids = all_result_ids + result_index;
num_queued++;
@@ -3236,66 +3256,21 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
// use `num_processed` since < `facet_batches.size()` batches could be processed (uneven splitting of records)
for(size_t batch_index = 0; batch_index < num_processed; batch_index++) {
auto& facet_batch = facet_batches[batch_index];
- for(size_t fi = 0; fi < facet_batch.size(); fi++) {
- auto& this_facet = facet_batch[fi];
- auto& acc_facet = facets[fi];
-
- acc_facet.is_intersected = this_facet.is_intersected;
- acc_facet.is_sort_by_alpha = this_facet.is_sort_by_alpha;
- acc_facet.sort_order = this_facet.sort_order;
- acc_facet.sort_field = this_facet.sort_field;
-
- for(auto & facet_kv: this_facet.result_map) {
- uint32_t fhash = 0;
- if(group_limit) {
- fhash = facet_kv.first;
- // we have to add all group sets
- acc_facet.hash_groups[fhash].insert(
- this_facet.hash_groups[fhash].begin(),
- this_facet.hash_groups[fhash].end()
- );
- } else {
- size_t count = 0;
- if (acc_facet.result_map.count(facet_kv.first) == 0) {
- // not found, so set it
- count = facet_kv.second.count;
- } else {
- count = acc_facet.result_map[facet_kv.first].count + facet_kv.second.count;
- }
- acc_facet.result_map[facet_kv.first].count = count;
- }
-
- acc_facet.result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;
- acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;
- acc_facet.result_map[facet_kv.first].sort_field_val = facet_kv.second.sort_field_val;
-
- acc_facet.hash_tokens[facet_kv.first] = this_facet.hash_tokens[facet_kv.first];
- }
-
- for(auto& facet_kv: this_facet.value_result_map) {
- size_t count = 0;
- if(acc_facet.value_result_map.count(facet_kv.first) == 0) {
- // not found, so set it
- count = facet_kv.second.count;
- } else {
- count = acc_facet.value_result_map[facet_kv.first].count + facet_kv.second.count;
- }
-
- acc_facet.value_result_map[facet_kv.first].count = count;
-
- acc_facet.value_result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;
- acc_facet.value_result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;
+ for(auto& this_facet : facet_batch) {
+ auto& acc_facet = facets[this_facet.orig_index];
+ aggregate_facet(group_limit, this_facet, acc_facet);
+ }
+ }
- acc_facet.fvalue_tokens[facet_kv.first] = this_facet.fvalue_tokens[facet_kv.first];
- }
+ // do value based faceting which must be single thread on the entire resultset (without batching)
+ do_facets(value_facets, facet_query, estimate_facets, facet_sample_percent,
+ facet_infos, group_limit, group_by_fields, group_missing_values,
+ all_result_ids, all_result_ids_len, max_facet_values,
+ is_wildcard_no_filter_query, facet_index_type);
- if(this_facet.stats.fvcount != 0) {
- acc_facet.stats.fvcount += this_facet.stats.fvcount;
- acc_facet.stats.fvsum += this_facet.stats.fvsum;
- acc_facet.stats.fvmax = std::max(acc_facet.stats.fvmax, this_facet.stats.fvmax);
- acc_facet.stats.fvmin = std::min(acc_facet.stats.fvmin, this_facet.stats.fvmin);
- }
- }
+ for(auto& this_facet : value_facets) {
+ auto& acc_facet = facets[this_facet.orig_index];
+ aggregate_facet(group_limit, this_facet, acc_facet);
}
for(auto & acc_facet: facets) {
@@ -3324,6 +3299,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
std::chrono::high_resolution_clock::now() - beginF).count();
LOG(INFO) << "Time for faceting: " << timeMillisF;*/
}
+
std::vector<facet_info_t> facet_infos(facets.size());
compute_facet_infos(facets, facet_query, facet_query_num_typos,
&included_ids_vec[0], included_ids_vec.size(), group_by_fields,
@@ -3345,6 +3321,64 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
return Option(true);
}
+void Index::aggregate_facet(const size_t group_limit, facet& this_facet, facet& acc_facet) const {
+ acc_facet.is_intersected = this_facet.is_intersected;
+ acc_facet.is_sort_by_alpha = this_facet.is_sort_by_alpha;
+ acc_facet.sort_order = this_facet.sort_order;
+ acc_facet.sort_field = this_facet.sort_field;
+
+ for(auto & facet_kv: this_facet.result_map) {
+ uint32_t fhash = 0;
+ if(group_limit) {
+ fhash = facet_kv.first;
+ // we have to add all group sets
+ acc_facet.hash_groups[fhash].insert(
+ this_facet.hash_groups[fhash].begin(),
+ this_facet.hash_groups[fhash].end()
+ );
+ } else {
+ size_t count = 0;
+ if (acc_facet.result_map.count(facet_kv.first) == 0) {
+ // not found, so set it
+ count = facet_kv.second.count;
+ } else {
+ count = acc_facet.result_map[facet_kv.first].count + facet_kv.second.count;
+ }
+ acc_facet.result_map[facet_kv.first].count = count;
+ }
+
+ acc_facet.result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;
+ acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;
+ acc_facet.result_map[facet_kv.first].sort_field_val = facet_kv.second.sort_field_val;
+
+ acc_facet.hash_tokens[facet_kv.first] = this_facet.hash_tokens[facet_kv.first];
+ }
+
+ for(auto& facet_kv: this_facet.value_result_map) {
+ size_t count = 0;
+ if(acc_facet.value_result_map.count(facet_kv.first) == 0) {
+ // not found, so set it
+ count = facet_kv.second.count;
+ } else {
+ count = acc_facet.value_result_map[facet_kv.first].count + facet_kv.second.count;
+ }
+
+ acc_facet.value_result_map[facet_kv.first].count = count;
+
+ acc_facet.value_result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;
+ acc_facet.value_result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;
+
+ acc_facet.fvalue_tokens[facet_kv.first] = this_facet.fvalue_tokens[facet_kv.first];
+ }
+
+ if(this_facet.stats.fvcount != 0) {
+ acc_facet.stats.fvcount += this_facet.stats.fvcount;
+ acc_facet.stats.fvsum += this_facet.stats.fvsum;
+ acc_facet.stats.fvmax = std::max(acc_facet.stats.fvmax, this_facet.stats.fvmax);
+ acc_facet.stats.fvmin = std::min(acc_facet.stats.fvmin, this_facet.stats.fvmin);
+ }
+}
+
void Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
const std::vector<uint32_t>& excluded_ids,
const std::vector<std::string>& group_by_fields, const size_t group_limit,
diff --git a/test/collection_optimized_faceting_test.cpp b/test/collection_optimized_faceting_test.cpp
index b2bcd84a..34199464 100644
--- a/test/collection_optimized_faceting_test.cpp
+++ b/test/collection_optimized_faceting_test.cpp
@@ -654,6 +654,56 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCountsFloatPrecision) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionOptimizedFacetingTest, FacetFloatStats) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::FLOAT, true)};
+
+ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
+ }
+
+ nlohmann::json doc;
+ doc["id"] = "100";
+ doc["title"] = "Ford Mustang";
+ doc["points"] = 50.4;
+ coll1->add(doc.dump());
+
+ doc["id"] = "200";
+ doc["title"] = "Ford Mustang";
+ doc["points"] = 50.4;
+ coll1->add(doc.dump());
+
+ std::vector<std::string> facets = {"points"};
+
+ nlohmann::json results = coll1->search("*", {"title"}, "", facets, sort_fields, {0}, 10, 1,
+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10,"", 30UL, 4UL,
+ "", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},
+ 4294967295UL, true, false, true, "", false, 6000000UL, 4UL,
+ 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,
+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();
+
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());
+
+ ASSERT_STREQ("points", results["facet_counts"][0]["field_name"].get<std::string>().c_str());
+ ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);
+
+ ASSERT_EQ(5, results["facet_counts"][0]["stats"].size());
+ ASSERT_FLOAT_EQ(50.40, results["facet_counts"][0]["stats"]["min"].get<double>());
+ ASSERT_FLOAT_EQ(50.40, results["facet_counts"][0]["stats"]["max"].get<double>());
+ ASSERT_FLOAT_EQ(100.80, results["facet_counts"][0]["stats"]["sum"].get<double>());
+ ASSERT_FLOAT_EQ(50.40, results["facet_counts"][0]["stats"]["avg"].get<double>());
+ ASSERT_FLOAT_EQ(1, results["facet_counts"][0]["stats"]["total_values"].get<size_t>());
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionOptimizedFacetingTest, FacetDeleteRepeatingValuesInArray) {
Collection *coll1;