Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.25-join' into v0.26-changes

Kishore Nallanbffaaf22fa78
diff --cc src/index.cpp
index 49143dfc,c7a33928..89874046
--- a/src/index.cpp
+++ b/src/index.cpp
@@@ -1198,170 -1151,92 +1198,172 @@@ void Index::do_facets(std::vector<facet
const auto& fquery_hashes = facet_infos[findex].hashes;
const bool should_compute_stats = facet_infos[findex].should_compute_stats;
+ auto sort_index_it = sort_index.find(a_facet.field_name);
+
size_t mod_value = 100 / facet_sample_percent;
- facet_map_t::iterator facet_map_it;
- single_val_facet_map_t::iterator single_facet_map_it;
- uint64_t fhash = 0;
- size_t facet_hash_count = 1;
- const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);
- const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);
+ auto facet_records = 0;
+
+ if(!facet_field.name.empty()) {
+ if(facet_field.is_string()) {
+ facet_records = facet_index_v4->get_facet_count(a_facet.field_name);
+ } else {
+ auto numerical_index_it = numerical_index.find(a_facet.field_name);
+ if(numerical_index_it != numerical_index.end()) {
+ facet_records = numerical_index_it->second->counter_list_size();
+ } else {
+ LOG(ERROR) << "facet " << a_facet.field_name << " not found in numerical index";
+ }
+ }
+ }
+
+ bool use_hashes = false;
- if((field_facet_mapping_it == facet_index_v3.end())
- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {
- continue;
+ if(!use_facet_intersection) {
+ use_hashes = true;
}
- for(size_t i = 0; i < results_size; i++) {
- // if sampling is enabled, we will skip a portion of the results to speed up things
- if(estimate_facets) {
- if(i % mod_value != 0) {
- continue;
+#ifndef TEST_BUILD
+ // non-test build should not accidentally set this flag
+ use_facet_intersection = false;
+ use_hashes = false;
+#endif
+
+ if(results_size && facet_records && ((facet_records <= 10 || is_wildcard_query) &&
+ !use_facet_query && group_limit == 0)
+ && !use_hashes || use_facet_intersection) {
+ //LOG(INFO) << "Using intersection to find facets";
+ a_facet.is_intersected = true;
+
+ std::map<std::string, uint32_t> facet_results;
+ if(!facet_field.name.empty()) {
+ if(facet_field.is_string()) {
+ facet_index_v4->intersect(a_facet.field_name, result_ids,
+ results_size, max_facet_count, facet_results, is_wildcard_query & no_filters_provided);
+ } else {
+ std::map<int64_t, uint32_t> facet_counts;
+ numerical_index.at(a_facet.field_name)->intersect(result_ids,
+ results_size, max_facet_count, facet_counts, is_wildcard_query & no_filters_provided);
+
+ for(const auto& kv : facet_counts) {
+ std::string val;
+ if(facet_field.is_float()) {
+ val = StringUtils::float_to_str(int64_t_to_float(kv.first));
+ } else if(facet_field.is_bool()) {
+ val = kv.first == 1 ? "true" : "false";
+ } else {
+ val = std::to_string(kv.first);
+ }
+
+ facet_results[val] = kv.second;
+ }
}
}
-
- uint32_t doc_seq_id = result_ids[i];
- if(facet_field.is_array()) {
- const auto& field_facet_mapping = field_facet_mapping_it->second;
- facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
-
- if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
- continue;
+
+ for(const auto& kv : facet_results) {
+ //range facet processing
+ if(a_facet.is_range_query) {
+ const auto doc_val = kv.first;
+ std::pair<std::string, std::string> range_pair {};
+ if(a_facet.get_range(doc_val, range_pair)) {
+ const auto& range_id = range_pair.first;
+ facet_count_t& facet_count = a_facet.result_map[range_id];
+ facet_count.count = kv.second;
+ }
+ } else {
+ facet_count_t& facet_count = a_facet.result_map[kv.first];
+ facet_count.count = kv.second;
}
- facet_hash_count = facet_map_it->second.size();
- } else {
- const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;
- single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
-
- if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
+
+ if(should_compute_stats) {
+ //LOG(INFO) << "Computing facet stas for facet " << a_facet.field_name;
+ for(int i = 0; i < kv.second; ++i) {
+ compute_facet_stats(a_facet, kv.first, facet_field.type);
+ }
+ }
+ }
+ } else {
+ facet_map_t::iterator facet_map_it;
+ single_val_facet_map_t::iterator single_facet_map_it;
+ uint32_t fhash = 0;
+ size_t facet_hash_count = 1;
+
+ const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);
+ const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);
+
+ if((field_facet_mapping_it == facet_index_v3.end())
+ && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {
continue;
- }
- facet_hash_count = 1;
- fhash = single_facet_map_it->second;
}
-
- const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;
- if(((i + 1) % 16384) == 0) {
- RETURN_CIRCUIT_BREAKER
- }
+ auto sort_index_it = sort_index.find(a_facet.field_name);
- for(size_t j = 0; j < facet_hash_count; j++) {
-
+ for(size_t i = 0; i < results_size; i++) {
+ // if sampling is enabled, we will skip a portion of the results to speed up things
+ if(estimate_facets) {
+ if(i % mod_value != 0) {
+ continue;
+ }
+ }
+
+ uint32_t doc_seq_id = result_ids[i];
if(facet_field.is_array()) {
- fhash = facet_map_it->second.hashes[j];
+ const auto& field_facet_mapping = field_facet_mapping_it->second;
+ facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
+
+ if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
+ continue;
+ }
+ facet_hash_count = facet_map_it->second.size();
+ } else {
+ const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;
+ single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
+
+ if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
+ continue;
+ }
+ facet_hash_count = 1;
+ fhash = single_facet_map_it->second;
}
-
- if(should_compute_stats) {
- compute_facet_stats(a_facet, fhash, facet_field.type);
+
+ const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;
+ //LOG(INFO) << "facet_hash_count " << facet_hash_count;
+ if(((i + 1) % 16384) == 0) {
+ RETURN_CIRCUIT_BREAKER
}
- if(a_facet.is_range_query) {
- if(sort_index_it != sort_index.end()){
- auto doc_id_val_map = sort_index_it->second;
- auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);
-
- if(doc_seq_id_it != doc_id_val_map->end()){
- int64_t doc_val = doc_seq_id_it->second;
- std::pair<int64_t, std::string> range_pair {};
- if(a_facet.get_range(doc_val, range_pair)) {
- int64_t range_id = range_pair.first;
- facet_count_t& facet_count = a_facet.result_map[range_id];
- facet_count.count += 1;
- }
- }
+
+ for(size_t j = 0; j < facet_hash_count; j++) {
+ if(facet_field.is_array()) {
+ fhash = facet_map_it->second.hashes[j];
}
- } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {
- facet_count_t& facet_count = a_facet.result_map[fhash];
- //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;
- facet_count.doc_id = doc_seq_id;
- facet_count.array_pos = j;
- if(group_limit) {
- a_facet.hash_groups[fhash].emplace(distinct_id);
- } else {
- facet_count.count += 1;
+ if(should_compute_stats) {
+ compute_facet_stats(a_facet, fhash, facet_field.type);
}
- if(use_facet_query) {
- a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);
+
+ if(a_facet.is_range_query) {
+ int64_t doc_val = get_doc_val_from_sort_index(sort_index_it, doc_seq_id);
+
+ std::pair<std::string, std::string> range_pair {};
+ if(a_facet.get_range(std::to_string(doc_val), range_pair)) {
+ const auto& range_id = range_pair.first;
+ facet_count_t& facet_count = a_facet.result_map[range_id];
+ facet_count.count += 1;
+ }
+ } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {
+ std::string fhash_str = std::to_string(fhash);
+ facet_count_t& facet_count = a_facet.result_map[fhash_str];
+ //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;
+ facet_count.doc_id = doc_seq_id;
+ facet_count.array_pos = j;
+ if(group_limit) {
+ a_facet.hash_groups[fhash].emplace(distinct_id);
+ } else {
+ facet_count.count += 1;
+ }
+ if(use_facet_query) {
+ //LOG (INFO) << "adding hash tokens for hash " << fhash;
+ a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);
+ }
}
}
}