Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

refactoring

krunal13137b59484e2f20
diff --git a/include/index.h b/include/index.h
index 342a0625..e3be3067 100644
--- a/include/index.h
+++ b/include/index.h
@@ -133,7 +133,7 @@ struct search_args {
const enable_t split_join_tokens;
tsl::htrie_map<char, token_leaf> qtoken_set;
- spp::sparse_hash_map<uint64_t, uint64_t> groups_processed;
+ spp::sparse_hash_map<uint64_t, uint32_t> groups_processed;
std::vector<std::vector<art_leaf*>> searched_queries;
Topster* topster;
Topster* curated_topster;
@@ -406,7 +406,7 @@ private:
int last_typo,
int max_typos,
std::vector<std::vector<art_leaf*>> & searched_queries,
- Topster* topster, spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
uint32_t** all_result_ids, size_t & all_result_ids_len,
size_t& field_num_results,
size_t group_limit,
@@ -433,7 +433,7 @@ private:
std::vector<std::vector<art_leaf*>>& searched_queries,
tsl::htrie_map<char, token_leaf>& qtoken_set,
Topster* topster,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
uint32_t*& all_result_ids, size_t& all_result_ids_len,
const size_t typo_tokens_threshold,
const size_t group_limit,
@@ -459,7 +459,7 @@ private:
const std::vector<uint32_t>& curated_ids,
std::vector<sort_by> & sort_fields, std::vector<token_candidates> & token_to_candidates,
std::vector<std::vector<art_leaf*>> & searched_queries,
- Topster* topster, spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
uint32_t** all_result_ids,
size_t & all_result_ids_len,
size_t& field_num_results,
@@ -603,7 +603,7 @@ public:
void score_results(const std::vector<sort_by> &sort_fields, const uint16_t &query_index, const uint8_t &field_id,
bool field_is_array, const uint32_t total_cost,
Topster *topster, const std::vector<art_leaf *> &query_suggestion,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
const uint32_t seq_id, const int sort_order[3],
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
const std::vector<size_t>& geopoint_indices,
@@ -660,7 +660,7 @@ public:
const size_t per_page,
const size_t page, const token_ordering token_order, const std::vector<bool>& prefixes,
const size_t drop_tokens_threshold, size_t& all_result_ids_len,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries,
tsl::htrie_map<char, token_leaf>& qtoken_set,
std::vector<std::vector<KV*>>& raw_result_kvs, std::vector<std::vector<KV*>>& override_result_kvs,
@@ -729,7 +729,7 @@ public:
void search_wildcard(filter_node_t const* const& filter_tree_root,
const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,
const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
@@ -781,7 +781,7 @@ public:
const std::vector<size_t>& geopoint_indices,
const std::vector<uint32_t>& curated_ids_sorted,
uint32_t*& all_result_ids, size_t& all_result_ids_len,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed) const;
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const;
void do_synonym_search(const std::vector<search_field_t>& the_fields,
const text_match_type_t match_type,
@@ -801,7 +801,7 @@ public:
Topster* actual_topster,
std::vector<std::vector<token_t>>& q_pos_synonyms,
int syn_orig_num_tokens,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries,
uint32_t*& all_result_ids, size_t& all_result_ids_len,
const uint32_t* filter_ids, uint32_t filter_ids_length,
@@ -827,7 +827,7 @@ public:
const std::vector<uint32_t>& num_typos,
std::vector<std::vector<art_leaf*>>& searched_queries,
tsl::htrie_map<char, token_leaf>& qtoken_set,
- Topster* topster, spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
uint32_t*& all_result_ids, size_t& all_result_ids_len,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
bool prioritize_exact_match,
@@ -863,7 +863,7 @@ public:
const text_match_type_t match_type,
const std::vector<sort_by>& sort_fields,
Topster* topster,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries,
tsl::htrie_map<char, token_leaf>& qtoken_set,
const size_t group_limit,
@@ -889,7 +889,7 @@ public:
const size_t min_typo, const std::vector<uint32_t>& num_typos,
Topster* topster, Topster* curated_topster, const token_ordering& token_order,
const std::vector<bool>& prefixes, const size_t drop_tokens_threshold,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries,
const size_t typo_tokens_threshold, const size_t group_limit,
const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
diff --git a/src/collection.cpp b/src/collection.cpp
index 38b9f5e4..7fb74f54 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1737,9 +1737,6 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
if(itr != search_params->groups_processed.end()) {
group_hits["found"] = itr->second;
}
- else {
- //LOG (ERROR) << "distinct key " << distinct_id << " not found ";
- }
result["grouped_hits"].push_back(group_hits);
}
}
diff --git a/src/index.cpp b/src/index.cpp
index ad299ddc..79f51504 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1404,7 +1404,7 @@ void Index::search_all_candidates(const size_t num_search_fields,
std::vector<std::vector<art_leaf*>>& searched_queries,
tsl::htrie_map<char, token_leaf>& qtoken_set,
Topster* topster,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
uint32_t*& all_result_ids, size_t& all_result_ids_len,
const size_t typo_tokens_threshold,
const size_t group_limit,
@@ -1481,7 +1481,7 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
std::vector<token_candidates> & token_candidates_vec,
std::vector<std::vector<art_leaf*>> & searched_queries,
Topster* topster,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
uint32_t** all_result_ids, size_t & all_result_ids_len,
size_t& field_num_results,
const size_t typo_tokens_threshold,
@@ -2334,7 +2334,7 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&
std::vector<facet> facets;
std::vector<std::vector<art_leaf*>> searched_queries;
Topster* topster = nullptr;
- spp::sparse_hash_map<uint64_t, uint64_t> groups_processed;
+ spp::sparse_hash_map<uint64_t, uint32_t> groups_processed;
uint32_t* result_ids = nullptr;
size_t result_ids_len = 0;
size_t field_num_results = 0;
@@ -2491,7 +2491,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
const size_t per_page,
const size_t page, const token_ordering token_order, const std::vector<bool>& prefixes,
const size_t drop_tokens_threshold, size_t& all_result_ids_len,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries,
tsl::htrie_map<char, token_leaf>& qtoken_set,
std::vector<std::vector<KV*>>& raw_result_kvs, std::vector<std::vector<KV*>>& override_result_kvs,
@@ -2588,7 +2588,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
int ret = topster->add(&kv);
- if((group_limit != 0) && (ret < 2)) {
+ if(group_limit != 0 && ret < 2) {
groups_processed[distinct_id]++;
}
@@ -2687,7 +2687,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
int ret = topster->add(&kv);
- if((group_limit != 0) && (ret < 2)) {
+ if(group_limit != 0 && ret < 2) {
groups_processed[distinct_id]++;
}
nearest_ids.push_back(seq_id);
@@ -3105,7 +3105,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
const std::vector<uint32_t>& num_typos,
std::vector<std::vector<art_leaf*>> & searched_queries,
tsl::htrie_map<char, token_leaf>& qtoken_set,
- Topster* topster, spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
uint32_t*& all_result_ids, size_t & all_result_ids_len,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
bool prioritize_exact_match,
@@ -3501,7 +3501,7 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
const text_match_type_t match_type,
const std::vector<sort_by>& sort_fields,
Topster* topster,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries,
tsl::htrie_map<char, token_leaf>& qtoken_set,
const size_t group_limit,
@@ -3705,7 +3705,7 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
}
int ret = topster->add(&kv);
- if((group_limit != 0) && (ret < 2)) {
+ if(group_limit != 0 && ret < 2) {
groups_processed[distinct_id]++;
}
result_ids.push_back(seq_id);
@@ -4078,7 +4078,7 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
Topster* actual_topster,
std::vector<std::vector<token_t>>& q_pos_synonyms,
int syn_orig_num_tokens,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries,
uint32_t*& all_result_ids, size_t& all_result_ids_len,
const uint32_t* filter_ids, const uint32_t filter_ids_length,
@@ -4115,7 +4115,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
const std::vector<size_t>& geopoint_indices,
const std::vector<uint32_t>& curated_ids_sorted,
uint32_t*& all_result_ids, size_t& all_result_ids_len,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed) const {
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const {
for(size_t field_id = 0; field_id < num_search_fields; field_id++) {
auto& field_name = the_fields[field_id].name;
@@ -4176,7 +4176,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
int ret = actual_topster->add(&kv);
- if((group_limit != 0) && (ret < 2)) {
+ if(group_limit != 0 && ret < 2) {
groups_processed[distinct_id]++;
}
@@ -4312,7 +4312,7 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
std::vector<std::vector<art_leaf*>> searched_queries;
Topster* topster = nullptr;
- spp::sparse_hash_map<uint64_t, uint64_t> groups_processed;
+ spp::sparse_hash_map<uint64_t, uint32_t> groups_processed;
uint32_t* field_result_ids = nullptr;
size_t field_result_ids_len = 0;
size_t field_num_results = 0;
@@ -4430,7 +4430,7 @@ void Index::curate_filtered_ids(filter_node_t const* const& filter_tree_root, co
void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,
const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
@@ -4514,7 +4514,7 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
int ret = topsters[thread_id]->add(&kv);
- if((group_limit != 0) && (ret < 2)) {
+ if(group_limit != 0 && ret < 2) {
tgroups_processed[thread_id][distinct_id]++;
}
@@ -4603,7 +4603,7 @@ void Index::search_field(const uint8_t & field_id,
const int last_typo,
const int max_typos,
std::vector<std::vector<art_leaf*>> & searched_queries,
- Topster* topster, spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
uint32_t** all_result_ids, size_t & all_result_ids_len, size_t& field_num_results,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
bool prioritize_exact_match,
@@ -4903,7 +4903,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
const uint8_t & field_id, const bool field_is_array, const uint32_t total_cost,
Topster* topster,
const std::vector<art_leaf *> &query_suggestion,
- spp::sparse_hash_map<uint64_t, uint64_t>& groups_processed,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
const uint32_t seq_id, const int sort_order[3],
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
const std::vector<size_t>& geopoint_indices,
@@ -5106,7 +5106,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
//LOG(INFO) << "Seq id: " << seq_id << ", match_score: " << match_score;
KV kv(query_index, seq_id, distinct_id, match_score_index, scores);
int ret = topster->add(&kv);
- if((group_limit != 0) && (ret < 2)) {
+ if(group_limit != 0 && ret < 2) {
groups_processed[distinct_id]++;
}