Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
updated conditions for wildcard results
krunal1313de9692deb535
diff --git a/include/facet_index.h b/include/facet_index.hindex 29243f8d..496c096c 100644--- a/include/facet_index.h+++ b/include/facet_index.h@@ -52,7 +52,7 @@ public:size_t get_facet_count(const std::string& field);int intersect(const std::string& val, const uint32_t* result_ids, int result_id_len,- int max_facet_count, std::map<std::string, uint32_t>& found);+ int max_facet_count, std::map<std::string, uint32_t>& found, bool is_wildcard_no_filter_query);std::string get_facet_by_count_index(const std::string& field, uint32_t count_index);};\ No newline at end of filediff --git a/include/index.h b/include/index.hindex 953dcf6d..d56c610c 100644--- a/include/index.h+++ b/include/index.h@@ -367,7 +367,7 @@ private:const std::vector<facet_info_t>& facet_infos,size_t group_limit, const std::vector<std::string>& group_by_fields,const uint32_t* result_ids, size_t results_size,- int max_facet_count) const;+ int max_facet_count, bool is_wildcard_query, bool no_filters_provided) const;bool static_filter_query_eval(const override_t* override, std::vector<std::string>& tokens,filter_node_t*& filter_tree_root) const;diff --git a/src/collection.cpp b/src/collection.cppindex b4e5849d..f990e48f 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1934,31 +1934,27 @@ Option<nlohmann::json> Collection::search(std::string raw_query,LOG (ERROR) << "range_id not found in result map.";}}- }-- if(a_facet.is_intersected) {- std::vector<std::pair<std::string, facet_count_t>> facet_counts;+ } else if(a_facet.is_intersected) {+ LOG(INFO) << "used intersection";+ std::vector<std::pair<std::string, uint32_t>> facet_counts;for (const auto & kv : a_facet.result_map) {- facet_counts.emplace_back(std::make_pair(kv.first, kv.second));+ facet_counts.emplace_back(std::make_pair(kv.first, kv.second.count));}auto max_facets = std::min(max_facet_values, facet_counts.size());std::sort(facet_counts.begin(), facet_counts.end(),[&](const auto& p1, const auto& p2) {- return p1.second.count > p2.second.count;+ return p1.second > p2.second;});for(int i = 0; i < max_facets; ++i) {- if(a_facet.is_range_query) {- break;- }const auto& kv = facet_counts[i];- facet_value_t facet_value = { kv.first, kv.first, kv.second.count};+ facet_value_t facet_value = { kv.first, kv.first, kv.second};facet_values.emplace_back(facet_value);}} else {-+ LOG(INFO) << "used hashes";std::vector<std::pair<uint32_t, facet_count_t>> facet_hash_counts;for (const auto & kv : a_facet.result_map) {@@ -1973,11 +1969,6 @@ Option<nlohmann::json> Collection::search(std::string raw_query,facet_hash_counts.end(), Collection::facet_count_compare);for(size_t fi = 0; fi < max_facets; fi++) {-- if(a_facet.is_range_query){- break;- }-// remap facet value hash with actual stringauto & kv = facet_hash_counts[fi];auto & facet_count = kv.second;diff --git a/src/facet_index.cpp b/src/facet_index.cppindex 0421653a..7875f091 100644--- a/src/facet_index.cpp+++ b/src/facet_index.cpp@@ -45,7 +45,7 @@ uint32_t facet_index_t::insert(const std::string& field, const std::string& valufor(it = counter_list.begin(); it != counter_list.end(); ++it) {// LOG (INFO) << "inserting in middle or front facet " << node.facet_value// << " with count " << node.count;- if(it->count >= facet_count) {+ if(it->count <= facet_count) {counter_list.emplace(it, node);break;}@@ -90,7 +90,7 @@ size_t facet_index_t::get_facet_count(const std::string& field) {//returns the count of matching seq_ids from result arrayint facet_index_t::intersect(const std::string& field, const uint32_t* result_ids,int result_ids_len, int max_facet_count,- std::map<std::string, uint32_t>& found) {+ std::map<std::string, uint32_t>& found, bool is_wildcard_no_filter_query) {//LOG (INFO) << "intersecting field " << field;const auto& facet_field_it = facet_field_map.find(field);@@ -107,27 +107,29 @@ int facet_index_t::intersect(const std::string& field, const uint32_t* result_idstd::vector<uint32_t> id_list;for(const auto& counter_list_it : counter_list) {// LOG (INFO) << "checking ids in facet_value " << counter_list_it.facet_value- // << " having total count " << counter_list_it.count;-- auto ids = facet_index_map.at(counter_list_it.facet_value).id_list_ptr;- ids_t::uncompress(ids, id_list);- const auto ids_len = id_list.size();+ // << " having total count " << counter_list_it.count;int count = 0;-- for(int i = 0; i < result_ids_len; ++i) {- if(std::binary_search(id_list.begin(), id_list.end(), result_ids[i])) {- ++count;++ if(is_wildcard_no_filter_query) {+ count = counter_list_it.count;+ } else {+ auto ids = facet_index_map.at(counter_list_it.facet_value).id_list_ptr;+ ids_t::uncompress(ids, id_list);+ const auto ids_len = id_list.size();+ for(int i = 0; i < result_ids_len; ++i) {+ if(std::binary_search(id_list.begin(), id_list.end(), result_ids[i])) {+ ++count;+ }}+ id_list.clear();}-+if(count) {found[counter_list_it.facet_value] = count;- if(found.size() == 2*max_facet_count) {+ if(found.size() == max_facet_count) {break;}}-- id_list.clear();}return found.size();diff --git a/src/index.cpp b/src/index.cppindex 1f12a6ad..99212649 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1214,7 +1214,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,const std::vector<facet_info_t>& facet_infos,const size_t group_limit, const std::vector<std::string>& group_by_fields,const uint32_t* result_ids, size_t results_size,- int max_facet_count) const {+ int max_facet_count, bool is_wildcard_query, bool no_filters_provided) const {// assumed that facet fields have already been validated upstreamfor(size_t findex=0; findex < facets.size(); findex++) {auto& a_facet = facets[findex];@@ -1228,19 +1228,15 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,size_t mod_value = 100 / facet_sample_percent;const auto facet_records = facet_index_v4->get_facet_count(a_facet.field_name);- const auto records_to_search = facet_records * results_size;- //LOG(INFO) << "results size " << results_size;- //LOG(INFO) << "records_to_search : " << records_to_search;- if(use_facet_query == false && group_limit == 0 && records_to_search != 0- && records_to_search < 10000) {+ if(results_size && facet_records && (facet_records <= 10 || is_wildcard_query == true)+ && use_facet_query == false && group_limit == 0 && no_filters_provided == true) {//LOG(INFO) << "Using intersection to find facets";a_facet.is_intersected = true;std::map<std::string, uint32_t> facet_results;facet_index_v4->intersect(a_facet.field_name, result_ids,- results_size, max_facet_count, facet_results);- //LOG(INFO) << "facet_results size " << facet_results.size();+ results_size, max_facet_count, facet_results, is_wildcard_query & no_filters_provided);for(const auto& kv : facet_results) {//range facet processing@@ -2409,7 +2405,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consauto is_wildcard_query = !field_query_tokens.empty() && !field_query_tokens[0].q_include_tokens.empty() &&field_query_tokens[0].q_include_tokens[0].value == "*";-+ bool no_filters_provided = (filter_tree_root == nullptr && filter_result.count == 0);+// handle phrase searchesif (!field_query_tokens[0].q_phrases.empty()) {@@ -2427,11 +2424,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consgoto process_search_results;}}-// for phrase query, parser will set field_query_tokens to "*", need to handle thatif (is_wildcard_query && field_query_tokens[0].q_phrases.empty()) {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);- bool no_filters_provided = (filter_tree_root == nullptr && !filter_result_iterator->is_valid);if(no_filters_provided && facets.empty() && curated_ids.empty() && vector_query.field_name.empty() &&sort_fields_std.size() == 1 && sort_fields_std[0].name == sort_field_const::seq_id &&@@ -2869,8 +2864,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consbool estimate_facets = (facet_sample_percent < 100 && all_result_ids_len > facet_sample_threshold);if(!facets.empty()) {- const size_t num_threads = std::min(concurrency, all_result_ids_len);- //const size_t num_threads = 1;+ //const size_t num_threads = std::min(concurrency, all_result_ids_len);+ const size_t num_threads = 1;+const size_t window_size = (num_threads == 0) ? 0 :(all_result_ids_len + num_threads - 1) / num_threads; // rounds upsize_t num_processed = 0;@@ -2910,7 +2906,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consthread_pool->enqueue([this, thread_id, &facet_batches, &facet_query, group_limit, group_by_fields,batch_result_ids, batch_res_len, &facet_infos, max_facet_values,- estimate_facets, facet_sample_percent,+ is_wildcard_query, no_filters_provided, estimate_facets, facet_sample_percent,&parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,&num_processed, &m_process, &cv_process]() {search_begin_us = parent_search_begin;@@ -2921,7 +2917,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consstd::vector<std::pair<std::string, uint32_t>> found_docs;do_facets(facet_batches[thread_id], fq, estimate_facets, facet_sample_percent,facet_infos, group_limit, group_by_fields,- batch_result_ids, batch_res_len, max_facet_values);+ batch_result_ids, batch_res_len, max_facet_values,+ is_wildcard_query, no_filters_provided);std::unique_lock<std::mutex> lock(m_process);num_processed++;parent_search_cutoff = parent_search_cutoff || search_cutoff;@@ -2962,6 +2959,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consacc_facet.result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;acc_facet.is_intersected = this_facet.is_intersected;+if(!facet_query.query.empty()) {fhash = std::stoul(facet_kv.first);//LOG(INFO) << "copying hash tokens for hash " << fhash;@@ -3005,7 +3003,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consmax_candidates, facet_infos);do_facets(facets, facet_query, estimate_facets, facet_sample_percent,facet_infos, group_limit, group_by_fields, &included_ids_vec[0],- included_ids_vec.size(), max_facet_values);+ included_ids_vec.size(), max_facet_values, is_wildcard_query, no_filters_provided);all_result_ids_len += curated_topster->size;diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex 8827c678..c9436bbd 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -747,8 +747,8 @@ TEST_F(CollectionFacetingTest, FacetCountOnSimilarStrings) {ASSERT_EQ(2, results["hits"].size());ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());- ASSERT_STREQ("England in India", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());- ASSERT_STREQ("India in England", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("India in England", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("England in India", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());collectionManager.drop_collection("coll1");}