Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Speed up token position computation.
kishorenc5795aba81fb4
diff --git a/include/collection.h b/include/collection.hindex 11c8fb19..f459c2a5 100644--- a/include/collection.h+++ b/include/collection.h@@ -172,7 +172,7 @@ private:return a.count > b.count;}- void free_leaf_indices(spp::sparse_hash_map<const art_leaf *, uint32_t *>& leaf_to_indices) const;+ void free_leaf_indices(std::vector<uint32_t*>& leaf_to_indices) const;public:Collection() = delete;diff --git a/include/index.h b/include/index.hindex b146a2c0..470e320a 100644--- a/include/index.h+++ b/include/index.h@@ -229,9 +229,9 @@ public:art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,- spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,+ std::vector<uint32_t*>& leaf_to_indices,size_t result_index,- std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions);+ std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> &array_token_positions);void score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,const uint32_t total_cost, Topster* topster, const std::vector<art_leaf *> & query_suggestion,@@ -280,9 +280,6 @@ public:search_args* search_params;- static void populate_array_token_positions(std::vector<std::vector<std::vector<uint16_t>>> & array_token_positions,- const art_leaf *token_leaf, uint32_t doc_index);-int get_bounded_typo_cost(const size_t max_cost, const size_t token_len) const;static int64_t float_to_in64_t(float n);diff --git a/src/collection.cpp b/src/collection.cppindex 72469b49..ca7b3215 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1214,7 +1214,7 @@ void Collection::highlight_result(const field &search_field,bool highlighted_fully,highlight_t & highlight) {- spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;+ std::vector<uint32_t*> leaf_to_indices;std::vector<art_leaf *> query_suggestion;for (const art_leaf *token_leaf : searched_queries[field_order_kv->query_index]) {@@ -1228,7 +1228,7 @@ void Collection::highlight_result(const field &search_field,uint32_t doc_index = actual_leaf->values->ids.indexOf(field_order_kv->key);auto doc_indices = new uint32_t[1];doc_indices[0] = doc_index;- leaf_to_indices.emplace(actual_leaf, doc_indices);+ leaf_to_indices.push_back(doc_indices);}}@@ -1239,13 +1239,14 @@ void Collection::highlight_result(const field &search_field,}// positions in the field of each token in the query- std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;+ std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> array_token_positions;Index::populate_token_positions(query_suggestion, leaf_to_indices, 0, array_token_positions);std::vector<match_index_t> match_indices;- for(size_t array_index = 0; array_index < array_token_positions.size(); array_index++) {- const std::vector<std::vector<uint16_t>> & token_positions = array_token_positions[array_index];+ for(const auto& kv: array_token_positions) {+ const std::vector<std::vector<uint16_t>>& token_positions = kv.second;+ size_t array_index = kv.first;if(token_positions.empty()) {continue;@@ -1346,10 +1347,9 @@ void Collection::highlight_result(const field &search_field,free_leaf_indices(leaf_to_indices);}-void Collection::free_leaf_indices(spp::sparse_hash_map<const art_leaf *, uint32_t *>& leaf_to_indices) const {- for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) {- delete [] it->second;- it->second = nullptr;+void Collection::free_leaf_indices(std::vector<uint32_t*>& leaf_to_indices) const {+ for(uint32_t* leaf_indices: leaf_to_indices) {+ delete [] leaf_indices;}}diff --git a/src/index.cpp b/src/index.cppindex cf59103a..94c380e8 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1030,18 +1030,18 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto}void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,- const std::vector<art_leaf *> &query_suggestion, uint32_t *exact_strt_ids,+ const std::vector<art_leaf *>& query_suggestion, uint32_t *exact_strt_ids,size_t& exact_strt_size) const {- spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;+ std::vector<uint32_t*> leaf_to_indices;for (art_leaf *token_leaf: query_suggestion) {if(token_leaf == nullptr) {- leaf_to_indices.emplace(token_leaf, nullptr);+ leaf_to_indices.push_back(nullptr);continue;}uint32_t *indices = new uint32_t[strt_ids_size];token_leaf->values->ids.indexOf(strt_ids, strt_ids_size, indices);- leaf_to_indices.emplace(token_leaf, indices);+ leaf_to_indices.push_back(indices);}// e.g. First In First Out => hash([0, 1, 0, 2])@@ -1059,12 +1059,12 @@ void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,}for(size_t strt_ids_index = 0; strt_ids_index < strt_ids_size; strt_ids_index++) {- std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;+ std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> array_token_positions;populate_token_positions(query_suggestion, leaf_to_indices, strt_ids_index, array_token_positions);// iterate array_token_positions and compute hash- for(size_t array_index = 0; array_index < array_token_positions.size(); array_index++) {- auto& token_positions = array_token_positions[array_index];+ for(const auto& kv: array_token_positions) {+ const std::vector<std::vector<uint16_t>>& token_positions = kv.second;size_t this_hash = 1;for(size_t token_index = 0; token_index < token_positions.size(); token_index++) {@@ -1506,11 +1506,11 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16spp::sparse_hash_set<uint64_t>& groups_processed,const uint32_t *result_ids, const size_t result_size) {- spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;- for (art_leaf *token_leaf : query_suggestion) {+ std::vector<uint32_t*> leaf_to_indices;+ for (art_leaf *token_leaf: query_suggestion) {uint32_t *indices = new uint32_t[result_size];token_leaf->values->ids.indexOf(result_ids, result_size, indices);- leaf_to_indices.emplace(token_leaf, indices);+ leaf_to_indices.push_back(indices);}std::unordered_map<std::string, size_t> facet_to_index;@@ -1543,10 +1543,11 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16if(query_suggestion.size() <= 1) {match_score = single_token_match_score;} else {- std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;+ std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> array_token_positions;populate_token_positions(query_suggestion, leaf_to_indices, i, array_token_positions);- for(const std::vector<std::vector<uint16_t>> & token_positions: array_token_positions) {+ for(const auto& kv: array_token_positions) {+ const std::vector<std::vector<uint16_t>>& token_positions = kv.second;if(token_positions.empty()) {continue;}@@ -1623,9 +1624,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();//LOG(INFO) << "Time taken for results iteration: " << timeNanos << "ms";- for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) {- delete [] it->second;- it->second = nullptr;+ for(uint32_t* leaf_indices: leaf_to_indices) {+ delete [] leaf_indices;}}@@ -1649,10 +1649,10 @@ uint64_t Index::get_distinct_id(const std::unordered_map<std::string, size_t> &freturn distinct_id;}-void Index::populate_token_positions(const std::vector<art_leaf *> &query_suggestion,- spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,+void Index::populate_token_positions(const std::vector<art_leaf *>& query_suggestion,+ std::vector<uint32_t*>& leaf_to_indices,size_t result_index,- std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions) {+ std::unordered_map<size_t, std::vector<std::vector<uint16_t>>>& array_token_positions) {if(query_suggestion.empty()) {return ;}@@ -1660,91 +1660,51 @@ void Index::populate_token_positions(const std::vector<art_leaf *> &query_sugges// array_token_positions:// for every element in a potential array, for every token in query suggestion, get the positions- // first ascertain the size of the array- size_t array_size = 0;-- for (const art_leaf *token_leaf : query_suggestion) {- size_t this_array_size = 1;+ for(size_t i = 0; i < query_suggestion.size(); i++) {+ const art_leaf* token_leaf = query_suggestion[i];+ uint32_t doc_index = leaf_to_indices[i][result_index];- uint32_t doc_index = leaf_to_indices.at(token_leaf)[result_index];+ // it's possible for a query token to not appear in a resulting documentif(doc_index == token_leaf->values->ids.getLength()) {continue;}+ // Array offset storage format:+ // a) last element is array_index b) second and third last elements will be largest offset+ // (last element is repeated to indicate end of offsets for a given array index)+uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?token_leaf->values->offsets.getLength() :token_leaf->values->offset_index.at(doc_index+1);- if(end_offset - start_offset < 3) {- this_array_size = 1; // can only be a string since array needs atleast 3 positions for storage- } else {- // Could be either an array or a string.- // Array offset storage format:- // a) last element is array_index b) second and third last elements will be largest offset-- auto last_val = (uint16_t) token_leaf->values->offsets.at(end_offset - 1);- auto second_last_val = (uint16_t) token_leaf->values->offsets.at(end_offset - 2);- auto third_last_val = (uint16_t) token_leaf->values->offsets.at(end_offset - 3);-- if(second_last_val != third_last_val) {- // guarantees that this is a string- this_array_size = 1;- } else {- this_array_size = last_val + 1;- }- }-- array_size = std::max(array_size, this_array_size);- }-- // initialize array_token_positions- array_token_positions = std::vector<std::vector<std::vector<uint16_t>>>(array_size);+ std::vector<uint16_t> positions;+ int prev_pos = -1;- // for each token in the query, find the positions that it appears in the array- for (const art_leaf *token_leaf : query_suggestion) {- uint32_t doc_index = leaf_to_indices.at(token_leaf)[result_index];- if(doc_index == token_leaf->values->ids.getLength()) {- continue;- }+ while(start_offset < end_offset) {+ int pos = token_leaf->values->offsets.at(start_offset);+ start_offset++;- populate_array_token_positions(array_token_positions, token_leaf, doc_index);- }-}+ if(pos == prev_pos) { // indicates end of array index+ if(!positions.empty()) {+ size_t array_index = (size_t) token_leaf->values->offsets.at(start_offset);+ array_token_positions[array_index].push_back(positions);+ positions.clear();+ }-void Index::populate_array_token_positions(std::vector<std::vector<std::vector<uint16_t>>> & array_token_positions,- const art_leaf *token_leaf, uint32_t doc_index) {- uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);- uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?- token_leaf->values->offsets.getLength() :- token_leaf->values->offset_index.at(doc_index+1);-- std::vector<uint16_t> positions;- uint16_t prev_pos = -1;-- while(start_offset < end_offset) {- auto pos = (uint16_t) token_leaf->values->offsets.at(start_offset);- start_offset++;-- if(pos == prev_pos) { // indicates end of array index- if(!positions.empty()) {- size_t array_index = (uint16_t) token_leaf->values->offsets.at(start_offset);- array_token_positions[array_index].push_back(positions);- positions.clear();+ start_offset++; // skip current value which is the array index+ prev_pos = -1;+ continue;}- start_offset++; // skip current value which is array index- prev_pos = -1;- continue;+ prev_pos = pos;+ positions.push_back((uint16_t)pos);}- prev_pos = pos;- positions.push_back(pos);- }-- if(!positions.empty()) {- // for plain string fields- array_token_positions[0].push_back(positions);+ if(!positions.empty()) {+ // for plain string fields+ array_token_positions[0].push_back(positions);+ }}}