Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

updated conditions for wildcard results

krunal1313de9692deb535
diff --git a/include/facet_index.h b/include/facet_index.h
index 29243f8d..496c096c 100644
--- a/include/facet_index.h
+++ b/include/facet_index.h
@@ -52,7 +52,7 @@ public:
size_t get_facet_count(const std::string& field);
int intersect(const std::string& val, const uint32_t* result_ids, int result_id_len,
- int max_facet_count, std::map<std::string, uint32_t>& found);
+ int max_facet_count, std::map<std::string, uint32_t>& found, bool is_wildcard_no_filter_query);
std::string get_facet_by_count_index(const std::string& field, uint32_t count_index);
};
\ No newline at end of file
diff --git a/include/index.h b/include/index.h
index 953dcf6d..d56c610c 100644
--- a/include/index.h
+++ b/include/index.h
@@ -367,7 +367,7 @@ private:
const std::vector<facet_info_t>& facet_infos,
size_t group_limit, const std::vector<std::string>& group_by_fields,
const uint32_t* result_ids, size_t results_size,
- int max_facet_count) const;
+ int max_facet_count, bool is_wildcard_query, bool no_filters_provided) const;
bool static_filter_query_eval(const override_t* override, std::vector<std::string>& tokens,
filter_node_t*& filter_tree_root) const;
diff --git a/src/collection.cpp b/src/collection.cpp
index b4e5849d..f990e48f 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1934,31 +1934,27 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
LOG (ERROR) << "range_id not found in result map.";
}
}
- }
-
- if(a_facet.is_intersected) {
- std::vector<std::pair<std::string, facet_count_t>> facet_counts;
+ } else if(a_facet.is_intersected) {
+ LOG(INFO) << "used intersection";
+ std::vector<std::pair<std::string, uint32_t>> facet_counts;
for (const auto & kv : a_facet.result_map) {
- facet_counts.emplace_back(std::make_pair(kv.first, kv.second));
+ facet_counts.emplace_back(std::make_pair(kv.first, kv.second.count));
}
auto max_facets = std::min(max_facet_values, facet_counts.size());
std::sort(facet_counts.begin(), facet_counts.end(),
[&](const auto& p1, const auto& p2) {
- return p1.second.count > p2.second.count;
+ return p1.second > p2.second;
});
for(int i = 0; i < max_facets; ++i) {
- if(a_facet.is_range_query) {
- break;
- }
const auto& kv = facet_counts[i];
- facet_value_t facet_value = { kv.first, kv.first, kv.second.count};
+ facet_value_t facet_value = { kv.first, kv.first, kv.second};
facet_values.emplace_back(facet_value);
}
} else {
-
+ LOG(INFO) << "used hashes";
std::vector<std::pair<uint32_t, facet_count_t>> facet_hash_counts;
for (const auto & kv : a_facet.result_map) {
@@ -1973,11 +1969,6 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
facet_hash_counts.end(), Collection::facet_count_compare);
for(size_t fi = 0; fi < max_facets; fi++) {
-
- if(a_facet.is_range_query){
- break;
- }
-
// remap facet value hash with actual string
auto & kv = facet_hash_counts[fi];
auto & facet_count = kv.second;
diff --git a/src/facet_index.cpp b/src/facet_index.cpp
index 0421653a..7875f091 100644
--- a/src/facet_index.cpp
+++ b/src/facet_index.cpp
@@ -45,7 +45,7 @@ uint32_t facet_index_t::insert(const std::string& field, const std::string& valu
for(it = counter_list.begin(); it != counter_list.end(); ++it) {
// LOG (INFO) << "inserting in middle or front facet " << node.facet_value
// << " with count " << node.count;
- if(it->count >= facet_count) {
+ if(it->count <= facet_count) {
counter_list.emplace(it, node);
break;
}
@@ -90,7 +90,7 @@ size_t facet_index_t::get_facet_count(const std::string& field) {
//returns the count of matching seq_ids from result array
int facet_index_t::intersect(const std::string& field, const uint32_t* result_ids,
int result_ids_len, int max_facet_count,
- std::map<std::string, uint32_t>& found) {
+ std::map<std::string, uint32_t>& found, bool is_wildcard_no_filter_query) {
//LOG (INFO) << "intersecting field " << field;
const auto& facet_field_it = facet_field_map.find(field);
@@ -107,27 +107,29 @@ int facet_index_t::intersect(const std::string& field, const uint32_t* result_id
std::vector<uint32_t> id_list;
for(const auto& counter_list_it : counter_list) {
// LOG (INFO) << "checking ids in facet_value " << counter_list_it.facet_value
- // << " having total count " << counter_list_it.count;
-
- auto ids = facet_index_map.at(counter_list_it.facet_value).id_list_ptr;
- ids_t::uncompress(ids, id_list);
- const auto ids_len = id_list.size();
+ // << " having total count " << counter_list_it.count;
int count = 0;
-
- for(int i = 0; i < result_ids_len; ++i) {
- if(std::binary_search(id_list.begin(), id_list.end(), result_ids[i])) {
- ++count;
+
+ if(is_wildcard_no_filter_query) {
+ count = counter_list_it.count;
+ } else {
+ auto ids = facet_index_map.at(counter_list_it.facet_value).id_list_ptr;
+ ids_t::uncompress(ids, id_list);
+ const auto ids_len = id_list.size();
+ for(int i = 0; i < result_ids_len; ++i) {
+ if(std::binary_search(id_list.begin(), id_list.end(), result_ids[i])) {
+ ++count;
+ }
}
+ id_list.clear();
}
-
+
if(count) {
found[counter_list_it.facet_value] = count;
- if(found.size() == 2*max_facet_count) {
+ if(found.size() == max_facet_count) {
break;
}
}
-
- id_list.clear();
}
return found.size();
diff --git a/src/index.cpp b/src/index.cpp
index 1f12a6ad..99212649 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1214,7 +1214,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
const std::vector<facet_info_t>& facet_infos,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
const uint32_t* result_ids, size_t results_size,
- int max_facet_count) const {
+ int max_facet_count, bool is_wildcard_query, bool no_filters_provided) const {
// assumed that facet fields have already been validated upstream
for(size_t findex=0; findex < facets.size(); findex++) {
auto& a_facet = facets[findex];
@@ -1228,19 +1228,15 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
size_t mod_value = 100 / facet_sample_percent;
const auto facet_records = facet_index_v4->get_facet_count(a_facet.field_name);
- const auto records_to_search = facet_records * results_size;
- //LOG(INFO) << "results size " << results_size;
- //LOG(INFO) << "records_to_search : " << records_to_search;
- if(use_facet_query == false && group_limit == 0 && records_to_search != 0
- && records_to_search < 10000) {
+ if(results_size && facet_records && (facet_records <= 10 || is_wildcard_query == true)
+ && use_facet_query == false && group_limit == 0 && no_filters_provided == true) {
//LOG(INFO) << "Using intersection to find facets";
a_facet.is_intersected = true;
std::map<std::string, uint32_t> facet_results;
facet_index_v4->intersect(a_facet.field_name, result_ids,
- results_size, max_facet_count, facet_results);
- //LOG(INFO) << "facet_results size " << facet_results.size();
+ results_size, max_facet_count, facet_results, is_wildcard_query & no_filters_provided);
for(const auto& kv : facet_results) {
//range facet processing
@@ -2409,7 +2405,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
auto is_wildcard_query = !field_query_tokens.empty() && !field_query_tokens[0].q_include_tokens.empty() &&
field_query_tokens[0].q_include_tokens[0].value == "*";
-
+ bool no_filters_provided = (filter_tree_root == nullptr && filter_result.count == 0);
+
// handle phrase searches
if (!field_query_tokens[0].q_phrases.empty()) {
@@ -2427,11 +2424,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
goto process_search_results;
}
}
-
// for phrase query, parser will set field_query_tokens to "*", need to handle that
if (is_wildcard_query && field_query_tokens[0].q_phrases.empty()) {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
- bool no_filters_provided = (filter_tree_root == nullptr && !filter_result_iterator->is_valid);
if(no_filters_provided && facets.empty() && curated_ids.empty() && vector_query.field_name.empty() &&
sort_fields_std.size() == 1 && sort_fields_std[0].name == sort_field_const::seq_id &&
@@ -2869,8 +2864,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
bool estimate_facets = (facet_sample_percent < 100 && all_result_ids_len > facet_sample_threshold);
if(!facets.empty()) {
- const size_t num_threads = std::min(concurrency, all_result_ids_len);
- //const size_t num_threads = 1;
+ //const size_t num_threads = std::min(concurrency, all_result_ids_len);
+ const size_t num_threads = 1;
+
const size_t window_size = (num_threads == 0) ? 0 :
(all_result_ids_len + num_threads - 1) / num_threads; // rounds up
size_t num_processed = 0;
@@ -2910,7 +2906,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
thread_pool->enqueue([this, thread_id, &facet_batches, &facet_query, group_limit, group_by_fields,
batch_result_ids, batch_res_len, &facet_infos, max_facet_values,
- estimate_facets, facet_sample_percent,
+ is_wildcard_query, no_filters_provided, estimate_facets, facet_sample_percent,
&parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,
&num_processed, &m_process, &cv_process]() {
search_begin_us = parent_search_begin;
@@ -2921,7 +2917,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
std::vector<std::pair<std::string, uint32_t>> found_docs;
do_facets(facet_batches[thread_id], fq, estimate_facets, facet_sample_percent,
facet_infos, group_limit, group_by_fields,
- batch_result_ids, batch_res_len, max_facet_values);
+ batch_result_ids, batch_res_len, max_facet_values,
+ is_wildcard_query, no_filters_provided);
std::unique_lock<std::mutex> lock(m_process);
num_processed++;
parent_search_cutoff = parent_search_cutoff || search_cutoff;
@@ -2962,6 +2959,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
acc_facet.result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;
acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;
acc_facet.is_intersected = this_facet.is_intersected;
+
if(!facet_query.query.empty()) {
fhash = std::stoul(facet_kv.first);
//LOG(INFO) << "copying hash tokens for hash " << fhash;
@@ -3005,7 +3003,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
max_candidates, facet_infos);
do_facets(facets, facet_query, estimate_facets, facet_sample_percent,
facet_infos, group_limit, group_by_fields, &included_ids_vec[0],
- included_ids_vec.size(), max_facet_values);
+ included_ids_vec.size(), max_facet_values, is_wildcard_query, no_filters_provided);
all_result_ids_len += curated_topster->size;
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index 8827c678..c9436bbd 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -747,8 +747,8 @@ TEST_F(CollectionFacetingTest, FacetCountOnSimilarStrings) {
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
- ASSERT_STREQ("England in India", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
- ASSERT_STREQ("India in England", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("India in England", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("England in India", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}