Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor query index logic.

Kishore Nallan71f1fbb4aa86
diff --git a/include/index.h b/include/index.h
index 291f0a76..26cc9f42 100644
--- a/include/index.h
+++ b/include/index.h
@@ -96,7 +96,7 @@ private:
void search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,
const std::vector<sort_by> & sort_fields, std::vector<token_candidates> & token_to_candidates,
const token_ordering token_order, std::vector<std::vector<art_leaf*>> & searched_queries,
- Topster<512> & topster, size_t & total_results, uint32_t** all_result_ids,
+ Topster<512> & topster, uint32_t** all_result_ids,
size_t & all_result_ids_len, const size_t & max_results, const bool prefix);
void insert_doc(const uint32_t score, art_tree *t, uint32_t seq_id,
@@ -152,7 +152,7 @@ public:
size_t result_index,
std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions);
- void score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const uint8_t & field_id,
+ void score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,
const uint32_t total_cost, Topster<512> &topster, const std::vector<art_leaf *> & query_suggestion,
const uint32_t *result_ids, const size_t result_size) const;
diff --git a/include/topster.h b/include/topster.h
index 52191547..759f1714 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -14,8 +14,8 @@
template <size_t MAX_SIZE=512>
struct Topster {
struct KV {
- uint8_t query_index;
uint8_t field_id;
+ uint16_t query_index;
uint64_t key;
uint64_t match_score;
number_t primary_attr;
@@ -36,7 +36,7 @@ struct Topster {
b = c;
}
- void add(const uint64_t &key, const uint8_t &query_index, const uint8_t &field_id, const uint64_t &match_score,
+ void add(const uint64_t &key, const uint8_t &field_id, const uint16_t &query_index, const uint64_t &match_score,
const number_t &primary_attr, const number_t &secondary_attr) {
if (size >= MAX_SIZE) {
if(!is_greater(data[0], match_score, primary_attr, secondary_attr)) {
@@ -53,8 +53,8 @@ struct Topster {
dedup_keys.insert(key);
data[0].key = key;
- data[0].query_index = query_index;
data[0].field_id = field_id;
+ data[0].query_index = query_index;
data[0].match_score = match_score;
data[0].primary_attr = primary_attr;
data[0].secondary_attr = secondary_attr;
@@ -84,8 +84,8 @@ struct Topster {
dedup_keys.insert(key);
data[size].key = key;
- data[size].query_index = query_index;
data[size].field_id = field_id;
+ data[size].query_index = query_index;
data[size].match_score = match_score;
data[size].primary_attr = primary_attr;
data[size].secondary_attr = secondary_attr;
diff --git a/src/collection.cpp b/src/collection.cpp
index 00308aaf..9d4f48b3 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -549,10 +549,6 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
for (const art_leaf *token_leaf : searched_queries[field_order_kv.query_index]) {
std::vector<uint16_t> positions;
uint32_t doc_index = token_leaf->values->ids.indexOf(field_order_kv.key);
- if(doc_index == token_leaf->values->ids.getLength()) {
- continue;
- }
-
uint32_t *indices = new uint32_t[1];
indices[0] = doc_index;
leaf_to_indices.emplace(token_leaf, indices);
diff --git a/src/index.cpp b/src/index.cpp
index 9909bffc..8f5bab2f 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -350,7 +350,7 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, si
const std::vector<sort_by> & sort_fields,
std::vector<token_candidates> & token_candidates_vec, const token_ordering token_order,
std::vector<std::vector<art_leaf*>> & searched_queries, Topster<512> & topster,
- size_t & total_results, uint32_t** all_result_ids, size_t & all_result_ids_len,
+ uint32_t** all_result_ids, size_t & all_result_ids_len,
const size_t & max_results, const bool prefix) {
const long long combination_limit = 10;
@@ -388,6 +388,11 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, si
result_ids = out;
}
+ if(result_size == 0) {
+ delete[] result_ids;
+ continue;
+ }
+
if(filter_ids != nullptr) {
// intersect once again with filter ids
uint32_t* filtered_result_ids = nullptr;
@@ -403,7 +408,7 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, si
do_facets(facets, filtered_result_ids, filtered_results_size);
// go through each matching document id and calculate match score
- score_results(sort_fields, searched_queries.size(), field_id, total_cost, topster, query_suggestion,
+ score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,
filtered_result_ids, filtered_results_size);
delete[] filtered_result_ids;
@@ -417,15 +422,14 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, si
delete [] *all_result_ids;
*all_result_ids = new_all_result_ids;
- score_results(sort_fields, searched_queries.size(), field_id, total_cost, topster, query_suggestion,
+ score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,
result_ids, result_size);
delete[] result_ids;
}
- total_results += topster.size;
searched_queries.push_back(query_suggestion);
- if(total_results >= max_results) {
+ if(all_result_ids_len >= max_results) {
break;
}
}
@@ -668,8 +672,6 @@ void Index::search_field(const uint8_t & field_id, std::string & query, const st
const size_t max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;
- size_t total_results = topster.size;
-
// To prevent us from doing ART search repeatedly as we iterate through possible corrections
spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;
@@ -753,7 +755,8 @@ void Index::search_field(const uint8_t & field_id, std::string & query, const st
token_to_costs[token_index].erase(it);
// when no more costs are left for this token and `drop_tokens_threshold` is breached
- if(token_to_costs[token_index].empty() && topster.size >= drop_tokens_threshold) {
+ if(token_to_costs[token_index].empty() && all_result_ids_len >= drop_tokens_threshold) {
+ n = combination_limit; // to break outer loop
break;
}
@@ -778,10 +781,10 @@ void Index::search_field(const uint8_t & field_id, std::string & query, const st
if(token_candidates_vec.size() != 0 && token_candidates_vec.size() == tokens.size()) {
// If all tokens were found, go ahead and search for candidates with what we have so far
search_candidates(field_id, filter_ids, filter_ids_length, facets, sort_fields, token_candidates_vec,
- token_order, searched_queries, topster, total_results, all_result_ids, all_result_ids_len,
+ token_order, searched_queries, topster, all_result_ids, all_result_ids_len,
Index::SEARCH_LIMIT_NUM, prefix);
- if (total_results >= Index::SEARCH_LIMIT_NUM) {
+ if (all_result_ids_len >= Index::SEARCH_LIMIT_NUM) {
// If we don't find enough results, we continue outerloop (looking at tokens with greater cost)
break;
}
@@ -791,7 +794,7 @@ void Index::search_field(const uint8_t & field_id, std::string & query, const st
}
// When there are not enough overall results and atleast one token has results
- if(topster.size < drop_tokens_threshold && token_to_count.size() > 1) {
+ if(all_result_ids_len < drop_tokens_threshold && token_to_count.size() > 1) {
// Drop token with least hits and try searching again
std::string truncated_query;
@@ -829,7 +832,7 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vect
}
}
-void Index::score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const uint8_t & field_id,
+void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,
const uint32_t total_cost, Topster<512> & topster,
const std::vector<art_leaf *> &query_suggestion,
const uint32_t *result_ids, const size_t result_size) const {
@@ -937,7 +940,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &
const number_t & primary_rank_value = primary_rank_score * primary_rank_factor;
const number_t & secondary_rank_value = secondary_rank_score * secondary_rank_factor;
- topster.add(seq_id, query_index, field_id, match_score, primary_rank_value, secondary_rank_value);
+ topster.add(seq_id, field_id, query_index, match_score, primary_rank_value, secondary_rank_value);
}
//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
diff --git a/test/topster_test.cpp b/test/topster_test.cpp
index e4e865d2..e0dc5fed 100644
--- a/test/topster_test.cpp
+++ b/test/topster_test.cpp
@@ -6,31 +6,31 @@ TEST(TopsterTest, StoreMaxIntValuesWithoutRepetition) {
Topster<5> topster;
struct {
- uint8_t query_index;
uint8_t field_id;
+ uint16_t query_index;
uint64_t key;
uint64_t match_score;
int64_t primary_attr;
int64_t secondary_attr;
} data[14] = {
- {0, 1, 1, 10, 20, 30},
- {0, 1, 1, 10, 20, 30},
- {0, 1, 2, 4, 20, 30},
- {2, 1, 3, 7, 20, 30},
- {0, 1, 4, 11, 20, 30},
+ {1, 0, 1, 10, 20, 30},
+ {1, 0, 1, 10, 20, 30},
+ {1, 0, 2, 4, 20, 30},
+ {1, 2, 3, 7, 20, 30},
+ {1, 0, 4, 11, 20, 30},
{1, 1, 5, 9, 20, 30},
{1, 1, 5, 9, 20, 30},
{1, 1, 5, 9, 20, 30},
- {0, 1, 6, 6, 20, 30},
- {2, 1, 7, 6, 22, 30},
- {2, 1, 7, 6, 22, 30},
+ {1, 0, 6, 6, 20, 30},
+ {1, 2, 7, 6, 22, 30},
+ {1, 2, 7, 6, 22, 30},
{1, 1, 8, 9, 20, 30},
- {0, 1, 9, 8, 20, 30},
- {3, 1, 10, 5, 20, 30},
+ {1, 0, 9, 8, 20, 30},
+ {1, 3, 10, 5, 20, 30},
};
for(int i = 0; i < 14; i++) {
- topster.add(data[i].key, data[i].query_index, 1, data[i].match_score, data[i].primary_attr,
+ topster.add(data[i].key, 1, data[i].query_index, data[i].match_score, data[i].primary_attr,
data[i].secondary_attr);
}
@@ -68,7 +68,7 @@ TEST(TopsterTest, StoreMaxFloatValuesWithoutRepetition) {
};
for(int i = 0; i < 12; i++) {
- topster.add(data[i].key, data[i].query_index, 1, data[i].match_score, data[i].primary_attr,
+ topster.add(data[i].key, 1, data[i].query_index, data[i].match_score, data[i].primary_attr,
data[i].secondary_attr);
}