Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.25-join' into v0.26-changes
Kishore Nallanbffaaf22fa78
diff --cc src/index.cppindex 49143dfc,c7a33928..89874046--- a/src/index.cpp+++ b/src/index.cpp@@@ -1198,170 -1151,92 +1198,172 @@@ void Index::do_facets(std::vector<facetconst auto& fquery_hashes = facet_infos[findex].hashes;const bool should_compute_stats = facet_infos[findex].should_compute_stats;+ auto sort_index_it = sort_index.find(a_facet.field_name);+size_t mod_value = 100 / facet_sample_percent;- facet_map_t::iterator facet_map_it;- single_val_facet_map_t::iterator single_facet_map_it;- uint64_t fhash = 0;- size_t facet_hash_count = 1;- const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);- const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);+ auto facet_records = 0;++ if(!facet_field.name.empty()) {+ if(facet_field.is_string()) {+ facet_records = facet_index_v4->get_facet_count(a_facet.field_name);+ } else {+ auto numerical_index_it = numerical_index.find(a_facet.field_name);+ if(numerical_index_it != numerical_index.end()) {+ facet_records = numerical_index_it->second->counter_list_size();+ } else {+ LOG(ERROR) << "facet " << a_facet.field_name << " not found in numerical index";+ }+ }+ }++ bool use_hashes = false;- if((field_facet_mapping_it == facet_index_v3.end())- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {- continue;+ if(!use_facet_intersection) {+ use_hashes = true;}- for(size_t i = 0; i < results_size; i++) {- // if sampling is enabled, we will skip a portion of the results to speed up things- if(estimate_facets) {- if(i % mod_value != 0) {- continue;+#ifndef TEST_BUILD+ // non-test build should not accidentally set this flag+ use_facet_intersection = false;+ use_hashes = false;+#endif++ if(results_size && facet_records && ((facet_records <= 10 || is_wildcard_query) &&+ !use_facet_query && group_limit == 0)+ && !use_hashes || use_facet_intersection) {+ //LOG(INFO) << "Using intersection to find facets";+ a_facet.is_intersected = true;++ std::map<std::string, uint32_t> facet_results;+ if(!facet_field.name.empty()) {+ if(facet_field.is_string()) {+ facet_index_v4->intersect(a_facet.field_name, result_ids,+ results_size, max_facet_count, facet_results, is_wildcard_query & no_filters_provided);+ } else {+ std::map<int64_t, uint32_t> facet_counts;+ numerical_index.at(a_facet.field_name)->intersect(result_ids,+ results_size, max_facet_count, facet_counts, is_wildcard_query & no_filters_provided);++ for(const auto& kv : facet_counts) {+ std::string val;+ if(facet_field.is_float()) {+ val = StringUtils::float_to_str(int64_t_to_float(kv.first));+ } else if(facet_field.is_bool()) {+ val = kv.first == 1 ? "true" : "false";+ } else {+ val = std::to_string(kv.first);+ }++ facet_results[val] = kv.second;+ }}}-- uint32_t doc_seq_id = result_ids[i];- if(facet_field.is_array()) {- const auto& field_facet_mapping = field_facet_mapping_it->second;- facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);-- if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {- continue;++ for(const auto& kv : facet_results) {+ //range facet processing+ if(a_facet.is_range_query) {+ const auto doc_val = kv.first;+ std::pair<std::string, std::string> range_pair {};+ if(a_facet.get_range(doc_val, range_pair)) {+ const auto& range_id = range_pair.first;+ facet_count_t& facet_count = a_facet.result_map[range_id];+ facet_count.count = kv.second;+ }+ } else {+ facet_count_t& facet_count = a_facet.result_map[kv.first];+ facet_count.count = kv.second;}- facet_hash_count = facet_map_it->second.size();- } else {- const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;- single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);-- if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {++ if(should_compute_stats) {+ //LOG(INFO) << "Computing facet stas for facet " << a_facet.field_name;+ for(int i = 0; i < kv.second; ++i) {+ compute_facet_stats(a_facet, kv.first, facet_field.type);+ }+ }+ }+ } else {+ facet_map_t::iterator facet_map_it;+ single_val_facet_map_t::iterator single_facet_map_it;+ uint32_t fhash = 0;+ size_t facet_hash_count = 1;++ const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);+ const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);++ if((field_facet_mapping_it == facet_index_v3.end())+ && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {continue;- }- facet_hash_count = 1;- fhash = single_facet_map_it->second;}-- const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;- if(((i + 1) % 16384) == 0) {- RETURN_CIRCUIT_BREAKER- }+ auto sort_index_it = sort_index.find(a_facet.field_name);- for(size_t j = 0; j < facet_hash_count; j++) {-+ for(size_t i = 0; i < results_size; i++) {+ // if sampling is enabled, we will skip a portion of the results to speed up things+ if(estimate_facets) {+ if(i % mod_value != 0) {+ continue;+ }+ }++ uint32_t doc_seq_id = result_ids[i];if(facet_field.is_array()) {- fhash = facet_map_it->second.hashes[j];+ const auto& field_facet_mapping = field_facet_mapping_it->second;+ facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);++ if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {+ continue;+ }+ facet_hash_count = facet_map_it->second.size();+ } else {+ const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;+ single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);++ if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {+ continue;+ }+ facet_hash_count = 1;+ fhash = single_facet_map_it->second;}-- if(should_compute_stats) {- compute_facet_stats(a_facet, fhash, facet_field.type);++ const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;+ //LOG(INFO) << "facet_hash_count " << facet_hash_count;+ if(((i + 1) % 16384) == 0) {+ RETURN_CIRCUIT_BREAKER}- if(a_facet.is_range_query) {- if(sort_index_it != sort_index.end()){- auto doc_id_val_map = sort_index_it->second;- auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);-- if(doc_seq_id_it != doc_id_val_map->end()){- int64_t doc_val = doc_seq_id_it->second;- std::pair<int64_t, std::string> range_pair {};- if(a_facet.get_range(doc_val, range_pair)) {- int64_t range_id = range_pair.first;- facet_count_t& facet_count = a_facet.result_map[range_id];- facet_count.count += 1;- }- }++ for(size_t j = 0; j < facet_hash_count; j++) {+ if(facet_field.is_array()) {+ fhash = facet_map_it->second.hashes[j];}- } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {- facet_count_t& facet_count = a_facet.result_map[fhash];- //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;- facet_count.doc_id = doc_seq_id;- facet_count.array_pos = j;- if(group_limit) {- a_facet.hash_groups[fhash].emplace(distinct_id);- } else {- facet_count.count += 1;+ if(should_compute_stats) {+ compute_facet_stats(a_facet, fhash, facet_field.type);}- if(use_facet_query) {- a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);++ if(a_facet.is_range_query) {+ int64_t doc_val = get_doc_val_from_sort_index(sort_index_it, doc_seq_id);++ std::pair<std::string, std::string> range_pair {};+ if(a_facet.get_range(std::to_string(doc_val), range_pair)) {+ const auto& range_id = range_pair.first;+ facet_count_t& facet_count = a_facet.result_map[range_id];+ facet_count.count += 1;+ }+ } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {+ std::string fhash_str = std::to_string(fhash);+ facet_count_t& facet_count = a_facet.result_map[fhash_str];+ //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;+ facet_count.doc_id = doc_seq_id;+ facet_count.array_pos = j;+ if(group_limit) {+ a_facet.hash_groups[fhash].emplace(distinct_id);+ } else {+ facet_count.count += 1;+ }+ if(use_facet_query) {+ //LOG (INFO) << "adding hash tokens for hash " << fhash;+ a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);+ }}}}