Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Speed up token position computation.

kishorenc5795aba81fb4
diff --git a/include/collection.h b/include/collection.h
index 11c8fb19..f459c2a5 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -172,7 +172,7 @@ private:
return a.count > b.count;
}
- void free_leaf_indices(spp::sparse_hash_map<const art_leaf *, uint32_t *>& leaf_to_indices) const;
+ void free_leaf_indices(std::vector<uint32_t*>& leaf_to_indices) const;
public:
Collection() = delete;
diff --git a/include/index.h b/include/index.h
index b146a2c0..470e320a 100644
--- a/include/index.h
+++ b/include/index.h
@@ -229,9 +229,9 @@ public:
art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);
static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,
- spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,
+ std::vector<uint32_t*>& leaf_to_indices,
size_t result_index,
- std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions);
+ std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> &array_token_positions);
void score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,
const uint32_t total_cost, Topster* topster, const std::vector<art_leaf *> & query_suggestion,
@@ -280,9 +280,6 @@ public:
search_args* search_params;
- static void populate_array_token_positions(std::vector<std::vector<std::vector<uint16_t>>> & array_token_positions,
- const art_leaf *token_leaf, uint32_t doc_index);
-
int get_bounded_typo_cost(const size_t max_cost, const size_t token_len) const;
static int64_t float_to_in64_t(float n);
diff --git a/src/collection.cpp b/src/collection.cpp
index 72469b49..ca7b3215 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1214,7 +1214,7 @@ void Collection::highlight_result(const field &search_field,
bool highlighted_fully,
highlight_t & highlight) {
- spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
+ std::vector<uint32_t*> leaf_to_indices;
std::vector<art_leaf *> query_suggestion;
for (const art_leaf *token_leaf : searched_queries[field_order_kv->query_index]) {
@@ -1228,7 +1228,7 @@ void Collection::highlight_result(const field &search_field,
uint32_t doc_index = actual_leaf->values->ids.indexOf(field_order_kv->key);
auto doc_indices = new uint32_t[1];
doc_indices[0] = doc_index;
- leaf_to_indices.emplace(actual_leaf, doc_indices);
+ leaf_to_indices.push_back(doc_indices);
}
}
@@ -1239,13 +1239,14 @@ void Collection::highlight_result(const field &search_field,
}
// positions in the field of each token in the query
- std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;
+ std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> array_token_positions;
Index::populate_token_positions(query_suggestion, leaf_to_indices, 0, array_token_positions);
std::vector<match_index_t> match_indices;
- for(size_t array_index = 0; array_index < array_token_positions.size(); array_index++) {
- const std::vector<std::vector<uint16_t>> & token_positions = array_token_positions[array_index];
+ for(const auto& kv: array_token_positions) {
+ const std::vector<std::vector<uint16_t>>& token_positions = kv.second;
+ size_t array_index = kv.first;
if(token_positions.empty()) {
continue;
@@ -1346,10 +1347,9 @@ void Collection::highlight_result(const field &search_field,
free_leaf_indices(leaf_to_indices);
}
-void Collection::free_leaf_indices(spp::sparse_hash_map<const art_leaf *, uint32_t *>& leaf_to_indices) const {
- for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) {
- delete [] it->second;
- it->second = nullptr;
+void Collection::free_leaf_indices(std::vector<uint32_t*>& leaf_to_indices) const {
+ for(uint32_t* leaf_indices: leaf_to_indices) {
+ delete [] leaf_indices;
}
}
diff --git a/src/index.cpp b/src/index.cpp
index cf59103a..94c380e8 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1030,18 +1030,18 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
}
void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,
- const std::vector<art_leaf *> &query_suggestion, uint32_t *exact_strt_ids,
+ const std::vector<art_leaf *>& query_suggestion, uint32_t *exact_strt_ids,
size_t& exact_strt_size) const {
- spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
+ std::vector<uint32_t*> leaf_to_indices;
for (art_leaf *token_leaf: query_suggestion) {
if(token_leaf == nullptr) {
- leaf_to_indices.emplace(token_leaf, nullptr);
+ leaf_to_indices.push_back(nullptr);
continue;
}
uint32_t *indices = new uint32_t[strt_ids_size];
token_leaf->values->ids.indexOf(strt_ids, strt_ids_size, indices);
- leaf_to_indices.emplace(token_leaf, indices);
+ leaf_to_indices.push_back(indices);
}
// e.g. First In First Out => hash([0, 1, 0, 2])
@@ -1059,12 +1059,12 @@ void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,
}
for(size_t strt_ids_index = 0; strt_ids_index < strt_ids_size; strt_ids_index++) {
- std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;
+ std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> array_token_positions;
populate_token_positions(query_suggestion, leaf_to_indices, strt_ids_index, array_token_positions);
// iterate array_token_positions and compute hash
- for(size_t array_index = 0; array_index < array_token_positions.size(); array_index++) {
- auto& token_positions = array_token_positions[array_index];
+ for(const auto& kv: array_token_positions) {
+ const std::vector<std::vector<uint16_t>>& token_positions = kv.second;
size_t this_hash = 1;
for(size_t token_index = 0; token_index < token_positions.size(); token_index++) {
@@ -1506,11 +1506,11 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
spp::sparse_hash_set<uint64_t>& groups_processed,
const uint32_t *result_ids, const size_t result_size) {
- spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
- for (art_leaf *token_leaf : query_suggestion) {
+ std::vector<uint32_t*> leaf_to_indices;
+ for (art_leaf *token_leaf: query_suggestion) {
uint32_t *indices = new uint32_t[result_size];
token_leaf->values->ids.indexOf(result_ids, result_size, indices);
- leaf_to_indices.emplace(token_leaf, indices);
+ leaf_to_indices.push_back(indices);
}
std::unordered_map<std::string, size_t> facet_to_index;
@@ -1543,10 +1543,11 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
if(query_suggestion.size() <= 1) {
match_score = single_token_match_score;
} else {
- std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;
+ std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> array_token_positions;
populate_token_positions(query_suggestion, leaf_to_indices, i, array_token_positions);
- for(const std::vector<std::vector<uint16_t>> & token_positions: array_token_positions) {
+ for(const auto& kv: array_token_positions) {
+ const std::vector<std::vector<uint16_t>>& token_positions = kv.second;
if(token_positions.empty()) {
continue;
}
@@ -1623,9 +1624,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
//LOG(INFO) << "Time taken for results iteration: " << timeNanos << "ms";
- for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) {
- delete [] it->second;
- it->second = nullptr;
+ for(uint32_t* leaf_indices: leaf_to_indices) {
+ delete [] leaf_indices;
}
}
@@ -1649,10 +1649,10 @@ uint64_t Index::get_distinct_id(const std::unordered_map<std::string, size_t> &f
return distinct_id;
}
-void Index::populate_token_positions(const std::vector<art_leaf *> &query_suggestion,
- spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,
+void Index::populate_token_positions(const std::vector<art_leaf *>& query_suggestion,
+ std::vector<uint32_t*>& leaf_to_indices,
size_t result_index,
- std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions) {
+ std::unordered_map<size_t, std::vector<std::vector<uint16_t>>>& array_token_positions) {
if(query_suggestion.empty()) {
return ;
}
@@ -1660,91 +1660,51 @@ void Index::populate_token_positions(const std::vector<art_leaf *> &query_sugges
// array_token_positions:
// for every element in a potential array, for every token in query suggestion, get the positions
- // first ascertain the size of the array
- size_t array_size = 0;
-
- for (const art_leaf *token_leaf : query_suggestion) {
- size_t this_array_size = 1;
+ for(size_t i = 0; i < query_suggestion.size(); i++) {
+ const art_leaf* token_leaf = query_suggestion[i];
+ uint32_t doc_index = leaf_to_indices[i][result_index];
- uint32_t doc_index = leaf_to_indices.at(token_leaf)[result_index];
+ // it's possible for a query token to not appear in a resulting document
if(doc_index == token_leaf->values->ids.getLength()) {
continue;
}
+ // Array offset storage format:
+ // a) last element is array_index b) second and third last elements will be largest offset
+ // (last element is repeated to indicate end of offsets for a given array index)
+
uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);
uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?
token_leaf->values->offsets.getLength() :
token_leaf->values->offset_index.at(doc_index+1);
- if(end_offset - start_offset < 3) {
- this_array_size = 1; // can only be a string since array needs atleast 3 positions for storage
- } else {
- // Could be either an array or a string.
- // Array offset storage format:
- // a) last element is array_index b) second and third last elements will be largest offset
-
- auto last_val = (uint16_t) token_leaf->values->offsets.at(end_offset - 1);
- auto second_last_val = (uint16_t) token_leaf->values->offsets.at(end_offset - 2);
- auto third_last_val = (uint16_t) token_leaf->values->offsets.at(end_offset - 3);
-
- if(second_last_val != third_last_val) {
- // guarantees that this is a string
- this_array_size = 1;
- } else {
- this_array_size = last_val + 1;
- }
- }
-
- array_size = std::max(array_size, this_array_size);
- }
-
- // initialize array_token_positions
- array_token_positions = std::vector<std::vector<std::vector<uint16_t>>>(array_size);
+ std::vector<uint16_t> positions;
+ int prev_pos = -1;
- // for each token in the query, find the positions that it appears in the array
- for (const art_leaf *token_leaf : query_suggestion) {
- uint32_t doc_index = leaf_to_indices.at(token_leaf)[result_index];
- if(doc_index == token_leaf->values->ids.getLength()) {
- continue;
- }
+ while(start_offset < end_offset) {
+ int pos = token_leaf->values->offsets.at(start_offset);
+ start_offset++;
- populate_array_token_positions(array_token_positions, token_leaf, doc_index);
- }
-}
+ if(pos == prev_pos) { // indicates end of array index
+ if(!positions.empty()) {
+ size_t array_index = (size_t) token_leaf->values->offsets.at(start_offset);
+ array_token_positions[array_index].push_back(positions);
+ positions.clear();
+ }
-void Index::populate_array_token_positions(std::vector<std::vector<std::vector<uint16_t>>> & array_token_positions,
- const art_leaf *token_leaf, uint32_t doc_index) {
- uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);
- uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?
- token_leaf->values->offsets.getLength() :
- token_leaf->values->offset_index.at(doc_index+1);
-
- std::vector<uint16_t> positions;
- uint16_t prev_pos = -1;
-
- while(start_offset < end_offset) {
- auto pos = (uint16_t) token_leaf->values->offsets.at(start_offset);
- start_offset++;
-
- if(pos == prev_pos) { // indicates end of array index
- if(!positions.empty()) {
- size_t array_index = (uint16_t) token_leaf->values->offsets.at(start_offset);
- array_token_positions[array_index].push_back(positions);
- positions.clear();
+ start_offset++; // skip current value which is the array index
+ prev_pos = -1;
+ continue;
}
- start_offset++; // skip current value which is array index
- prev_pos = -1;
- continue;
+ prev_pos = pos;
+ positions.push_back((uint16_t)pos);
}
- prev_pos = pos;
- positions.push_back(pos);
- }
-
- if(!positions.empty()) {
- // for plain string fields
- array_token_positions[0].push_back(positions);
+ if(!positions.empty()) {
+ // for plain string fields
+ array_token_positions[0].push_back(positions);
+ }
}
}