Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Buffer IDs during exhaustive search.
Kishore Nallan97b1981fcdda
diff --git a/include/index.h b/include/index.hindex 9c8d0fc4..1309abea 100644--- a/include/index.h+++ b/include/index.h@@ -510,7 +510,8 @@ private:bool prioritize_exact_match,bool exhaustive_search,size_t concurrency,- std::set<uint64>& query_hashes) const;+ std::set<uint64>& query_hashes,+ std::vector<uint32_t>& id_buff) const;void do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length, const std::vector<filter>& filters,const bool enable_short_circuit) const;diff --git a/include/posting.h b/include/posting.hindex c5c3ef66..e8ad4be3 100644--- a/include/posting.h+++ b/include/posting.h@@ -65,9 +65,11 @@ public:to_expanded_plists(raw_posting_lists, plists, expanded_plists);- std::sort(this->plists.begin(), this->plists.end(), [](posting_list_t* a, posting_list_t* b) {- return a->num_blocks() < b->num_blocks();- });+ if(plists.size() > 1) {+ std::sort(this->plists.begin(), this->plists.end(), [](posting_list_t* a, posting_list_t* b) {+ return a->num_blocks() < b->num_blocks();+ });+ }}~block_intersector_t() {diff --git a/src/index.cpp b/src/index.cppindex 418dc98b..1a677731 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1075,7 +1075,8 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,bool prioritize_exact_match,const bool exhaustive_search,const size_t concurrency,- std::set<uint64>& query_hashes) const {+ std::set<uint64>& query_hashes,+ std::vector<uint32_t>& id_buff) const {auto product = []( long long a, token_candidates & b ) { return a*b.candidates.size(); };long long int N = std::accumulate(token_candidates_vec.begin(), token_candidates_vec.end(), 1LL, product);@@ -1180,11 +1181,15 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,for(size_t i = 0; i < concurrency; i++) {// empty vec can happen if not all threads produce resultsif (!result_id_vecs[i].empty()) {- uint32_t* new_all_result_ids = nullptr;- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &result_id_vecs[i][0],- result_id_vecs[i].size(), &new_all_result_ids);- delete[] *all_result_ids;- *all_result_ids = new_all_result_ids;+ if(exhaustive_search) {+ id_buff.insert(id_buff.end(), result_id_vecs[i].begin(), result_id_vecs[i].end());+ } else {+ uint32_t* new_all_result_ids = nullptr;+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &result_id_vecs[i][0],+ result_id_vecs[i].size(), &new_all_result_ids);+ delete[] *all_result_ids;+ *all_result_ids = new_all_result_ids;+ }num_result_ids += result_id_vecs[i].size();@@ -1200,6 +1205,20 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,}}+ if(id_buff.size() > 100000) {+ // prevents too many ORs during exhaustive searching+ std::sort(id_buff.begin(), id_buff.end());+ id_buff.erase(std::unique( id_buff.begin(), id_buff.end() ), id_buff.end());++ uint32_t* new_all_result_ids = nullptr;+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &id_buff[0],+ id_buff.size(), &new_all_result_ids);+ delete[] *all_result_ids;+ *all_result_ids = new_all_result_ids;+ num_result_ids += id_buff.size();+ id_buff.clear();+ }+if(num_result_ids == 0) {continue;}@@ -2940,13 +2959,26 @@ void Index::search_field(const uint8_t & field_id,}if(!token_candidates_vec.empty()) {+ std::vector<uint32_t> id_buff;+// If atleast one token is found, go ahead and search for candidatessearch_candidates(field_id, the_field.is_array(), filter_ids, filter_ids_length,exclude_token_ids, exclude_token_ids_size,curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,groups_processed, all_result_ids, all_result_ids_len, field_num_results,typo_tokens_threshold, group_limit, group_by_fields, query_tokens,- prioritize_exact_match, combination_limit, concurrency, query_hashes);+ prioritize_exact_match, combination_limit, concurrency, query_hashes, id_buff);++ if(id_buff.size() > 1) {+ std::sort(id_buff.begin(), id_buff.end());+ id_buff.erase(std::unique( id_buff.begin(), id_buff.end() ), id_buff.end());+ }++ uint32_t* new_all_result_ids = nullptr;+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &id_buff[0],+ id_buff.size(), &new_all_result_ids);+ delete[] *all_result_ids;+ *all_result_ids = new_all_result_ids;}resume_typo_loop:diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 63ceba4f..09229ed0 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1939,7 +1939,7 @@ TEST_F(CollectionTest, DeletionOfDocumentArrayFields) {token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10).get();- ASSERT_EQ(1, res["found"]);+ ASSERT_EQ(1, res["found"].get<size_t>());Option<std::string> rem_op = coll1->remove("100");