Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor query index logic.
Kishore Nallan71f1fbb4aa86
diff --git a/include/index.h b/include/index.hindex 291f0a76..26cc9f42 100644--- a/include/index.h+++ b/include/index.h@@ -96,7 +96,7 @@ private:void search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,const std::vector<sort_by> & sort_fields, std::vector<token_candidates> & token_to_candidates,const token_ordering token_order, std::vector<std::vector<art_leaf*>> & searched_queries,- Topster<512> & topster, size_t & total_results, uint32_t** all_result_ids,+ Topster<512> & topster, uint32_t** all_result_ids,size_t & all_result_ids_len, const size_t & max_results, const bool prefix);void insert_doc(const uint32_t score, art_tree *t, uint32_t seq_id,@@ -152,7 +152,7 @@ public:size_t result_index,std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions);- void score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const uint8_t & field_id,+ void score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,const uint32_t total_cost, Topster<512> &topster, const std::vector<art_leaf *> & query_suggestion,const uint32_t *result_ids, const size_t result_size) const;diff --git a/include/topster.h b/include/topster.hindex 52191547..759f1714 100644--- a/include/topster.h+++ b/include/topster.h@@ -14,8 +14,8 @@template <size_t MAX_SIZE=512>struct Topster {struct KV {- uint8_t query_index;uint8_t field_id;+ uint16_t query_index;uint64_t key;uint64_t match_score;number_t primary_attr;@@ -36,7 +36,7 @@ struct Topster {b = c;}- void add(const uint64_t &key, const uint8_t &query_index, const uint8_t &field_id, const uint64_t &match_score,+ void add(const uint64_t &key, const uint8_t &field_id, const uint16_t &query_index, const uint64_t &match_score,const number_t &primary_attr, const number_t &secondary_attr) {if (size >= MAX_SIZE) {if(!is_greater(data[0], match_score, primary_attr, secondary_attr)) {@@ -53,8 +53,8 @@ struct Topster {dedup_keys.insert(key);data[0].key = key;- data[0].query_index = query_index;data[0].field_id = field_id;+ data[0].query_index = query_index;data[0].match_score = match_score;data[0].primary_attr = primary_attr;data[0].secondary_attr = secondary_attr;@@ -84,8 +84,8 @@ struct Topster {dedup_keys.insert(key);data[size].key = key;- data[size].query_index = query_index;data[size].field_id = field_id;+ data[size].query_index = query_index;data[size].match_score = match_score;data[size].primary_attr = primary_attr;data[size].secondary_attr = secondary_attr;diff --git a/src/collection.cpp b/src/collection.cppindex 00308aaf..9d4f48b3 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -549,10 +549,6 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<sfor (const art_leaf *token_leaf : searched_queries[field_order_kv.query_index]) {std::vector<uint16_t> positions;uint32_t doc_index = token_leaf->values->ids.indexOf(field_order_kv.key);- if(doc_index == token_leaf->values->ids.getLength()) {- continue;- }-uint32_t *indices = new uint32_t[1];indices[0] = doc_index;leaf_to_indices.emplace(token_leaf, indices);diff --git a/src/index.cpp b/src/index.cppindex 9909bffc..8f5bab2f 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -350,7 +350,7 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, siconst std::vector<sort_by> & sort_fields,std::vector<token_candidates> & token_candidates_vec, const token_ordering token_order,std::vector<std::vector<art_leaf*>> & searched_queries, Topster<512> & topster,- size_t & total_results, uint32_t** all_result_ids, size_t & all_result_ids_len,+ uint32_t** all_result_ids, size_t & all_result_ids_len,const size_t & max_results, const bool prefix) {const long long combination_limit = 10;@@ -388,6 +388,11 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, siresult_ids = out;}+ if(result_size == 0) {+ delete[] result_ids;+ continue;+ }+if(filter_ids != nullptr) {// intersect once again with filter idsuint32_t* filtered_result_ids = nullptr;@@ -403,7 +408,7 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, sido_facets(facets, filtered_result_ids, filtered_results_size);// go through each matching document id and calculate match score- score_results(sort_fields, searched_queries.size(), field_id, total_cost, topster, query_suggestion,+ score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,filtered_result_ids, filtered_results_size);delete[] filtered_result_ids;@@ -417,15 +422,14 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, sidelete [] *all_result_ids;*all_result_ids = new_all_result_ids;- score_results(sort_fields, searched_queries.size(), field_id, total_cost, topster, query_suggestion,+ score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,result_ids, result_size);delete[] result_ids;}- total_results += topster.size;searched_queries.push_back(query_suggestion);- if(total_results >= max_results) {+ if(all_result_ids_len >= max_results) {break;}}@@ -668,8 +672,6 @@ void Index::search_field(const uint8_t & field_id, std::string & query, const stconst size_t max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;- size_t total_results = topster.size;-// To prevent us from doing ART search repeatedly as we iterate through possible correctionsspp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;@@ -753,7 +755,8 @@ void Index::search_field(const uint8_t & field_id, std::string & query, const sttoken_to_costs[token_index].erase(it);// when no more costs are left for this token and `drop_tokens_threshold` is breached- if(token_to_costs[token_index].empty() && topster.size >= drop_tokens_threshold) {+ if(token_to_costs[token_index].empty() && all_result_ids_len >= drop_tokens_threshold) {+ n = combination_limit; // to break outer loopbreak;}@@ -778,10 +781,10 @@ void Index::search_field(const uint8_t & field_id, std::string & query, const stif(token_candidates_vec.size() != 0 && token_candidates_vec.size() == tokens.size()) {// If all tokens were found, go ahead and search for candidates with what we have so farsearch_candidates(field_id, filter_ids, filter_ids_length, facets, sort_fields, token_candidates_vec,- token_order, searched_queries, topster, total_results, all_result_ids, all_result_ids_len,+ token_order, searched_queries, topster, all_result_ids, all_result_ids_len,Index::SEARCH_LIMIT_NUM, prefix);- if (total_results >= Index::SEARCH_LIMIT_NUM) {+ if (all_result_ids_len >= Index::SEARCH_LIMIT_NUM) {// If we don't find enough results, we continue outerloop (looking at tokens with greater cost)break;}@@ -791,7 +794,7 @@ void Index::search_field(const uint8_t & field_id, std::string & query, const st}// When there are not enough overall results and atleast one token has results- if(topster.size < drop_tokens_threshold && token_to_count.size() > 1) {+ if(all_result_ids_len < drop_tokens_threshold && token_to_count.size() > 1) {// Drop token with least hits and try searching againstd::string truncated_query;@@ -829,7 +832,7 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vect}}-void Index::score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const uint8_t & field_id,+void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,const uint32_t total_cost, Topster<512> & topster,const std::vector<art_leaf *> &query_suggestion,const uint32_t *result_ids, const size_t result_size) const {@@ -937,7 +940,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &const number_t & primary_rank_value = primary_rank_score * primary_rank_factor;const number_t & secondary_rank_value = secondary_rank_score * secondary_rank_factor;- topster.add(seq_id, query_index, field_id, match_score, primary_rank_value, secondary_rank_value);+ topster.add(seq_id, field_id, query_index, match_score, primary_rank_value, secondary_rank_value);}//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --git a/test/topster_test.cpp b/test/topster_test.cppindex e4e865d2..e0dc5fed 100644--- a/test/topster_test.cpp+++ b/test/topster_test.cpp@@ -6,31 +6,31 @@ TEST(TopsterTest, StoreMaxIntValuesWithoutRepetition) {Topster<5> topster;struct {- uint8_t query_index;uint8_t field_id;+ uint16_t query_index;uint64_t key;uint64_t match_score;int64_t primary_attr;int64_t secondary_attr;} data[14] = {- {0, 1, 1, 10, 20, 30},- {0, 1, 1, 10, 20, 30},- {0, 1, 2, 4, 20, 30},- {2, 1, 3, 7, 20, 30},- {0, 1, 4, 11, 20, 30},+ {1, 0, 1, 10, 20, 30},+ {1, 0, 1, 10, 20, 30},+ {1, 0, 2, 4, 20, 30},+ {1, 2, 3, 7, 20, 30},+ {1, 0, 4, 11, 20, 30},{1, 1, 5, 9, 20, 30},{1, 1, 5, 9, 20, 30},{1, 1, 5, 9, 20, 30},- {0, 1, 6, 6, 20, 30},- {2, 1, 7, 6, 22, 30},- {2, 1, 7, 6, 22, 30},+ {1, 0, 6, 6, 20, 30},+ {1, 2, 7, 6, 22, 30},+ {1, 2, 7, 6, 22, 30},{1, 1, 8, 9, 20, 30},- {0, 1, 9, 8, 20, 30},- {3, 1, 10, 5, 20, 30},+ {1, 0, 9, 8, 20, 30},+ {1, 3, 10, 5, 20, 30},};for(int i = 0; i < 14; i++) {- topster.add(data[i].key, data[i].query_index, 1, data[i].match_score, data[i].primary_attr,+ topster.add(data[i].key, 1, data[i].query_index, data[i].match_score, data[i].primary_attr,data[i].secondary_attr);}@@ -68,7 +68,7 @@ TEST(TopsterTest, StoreMaxFloatValuesWithoutRepetition) {};for(int i = 0; i < 12; i++) {- topster.add(data[i].key, data[i].query_index, 1, data[i].match_score, data[i].primary_attr,+ topster.add(data[i].key, 1, data[i].query_index, data[i].match_score, data[i].primary_attr,data[i].secondary_attr);}