Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Buffer IDs during exhaustive search.

Kishore Nallan97b1981fcdda
diff --git a/include/index.h b/include/index.h
index 9c8d0fc4..1309abea 100644
--- a/include/index.h
+++ b/include/index.h
@@ -510,7 +510,8 @@ private:
bool prioritize_exact_match,
bool exhaustive_search,
size_t concurrency,
- std::set<uint64>& query_hashes) const;
+ std::set<uint64>& query_hashes,
+ std::vector<uint32_t>& id_buff) const;
void do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length, const std::vector<filter>& filters,
const bool enable_short_circuit) const;
diff --git a/include/posting.h b/include/posting.h
index c5c3ef66..e8ad4be3 100644
--- a/include/posting.h
+++ b/include/posting.h
@@ -65,9 +65,11 @@ public:
to_expanded_plists(raw_posting_lists, plists, expanded_plists);
- std::sort(this->plists.begin(), this->plists.end(), [](posting_list_t* a, posting_list_t* b) {
- return a->num_blocks() < b->num_blocks();
- });
+ if(plists.size() > 1) {
+ std::sort(this->plists.begin(), this->plists.end(), [](posting_list_t* a, posting_list_t* b) {
+ return a->num_blocks() < b->num_blocks();
+ });
+ }
}
~block_intersector_t() {
diff --git a/src/index.cpp b/src/index.cpp
index 418dc98b..1a677731 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1075,7 +1075,8 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
bool prioritize_exact_match,
const bool exhaustive_search,
const size_t concurrency,
- std::set<uint64>& query_hashes) const {
+ std::set<uint64>& query_hashes,
+ std::vector<uint32_t>& id_buff) const {
auto product = []( long long a, token_candidates & b ) { return a*b.candidates.size(); };
long long int N = std::accumulate(token_candidates_vec.begin(), token_candidates_vec.end(), 1LL, product);
@@ -1180,11 +1181,15 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
for(size_t i = 0; i < concurrency; i++) {
// empty vec can happen if not all threads produce results
if (!result_id_vecs[i].empty()) {
- uint32_t* new_all_result_ids = nullptr;
- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &result_id_vecs[i][0],
- result_id_vecs[i].size(), &new_all_result_ids);
- delete[] *all_result_ids;
- *all_result_ids = new_all_result_ids;
+ if(exhaustive_search) {
+ id_buff.insert(id_buff.end(), result_id_vecs[i].begin(), result_id_vecs[i].end());
+ } else {
+ uint32_t* new_all_result_ids = nullptr;
+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &result_id_vecs[i][0],
+ result_id_vecs[i].size(), &new_all_result_ids);
+ delete[] *all_result_ids;
+ *all_result_ids = new_all_result_ids;
+ }
num_result_ids += result_id_vecs[i].size();
@@ -1200,6 +1205,20 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
}
}
+ if(id_buff.size() > 100000) {
+ // prevents too many ORs during exhaustive searching
+ std::sort(id_buff.begin(), id_buff.end());
+ id_buff.erase(std::unique( id_buff.begin(), id_buff.end() ), id_buff.end());
+
+ uint32_t* new_all_result_ids = nullptr;
+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &id_buff[0],
+ id_buff.size(), &new_all_result_ids);
+ delete[] *all_result_ids;
+ *all_result_ids = new_all_result_ids;
+ num_result_ids += id_buff.size();
+ id_buff.clear();
+ }
+
if(num_result_ids == 0) {
continue;
}
@@ -2940,13 +2959,26 @@ void Index::search_field(const uint8_t & field_id,
}
if(!token_candidates_vec.empty()) {
+ std::vector<uint32_t> id_buff;
+
// If atleast one token is found, go ahead and search for candidates
search_candidates(field_id, the_field.is_array(), filter_ids, filter_ids_length,
exclude_token_ids, exclude_token_ids_size,
curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,
groups_processed, all_result_ids, all_result_ids_len, field_num_results,
typo_tokens_threshold, group_limit, group_by_fields, query_tokens,
- prioritize_exact_match, combination_limit, concurrency, query_hashes);
+ prioritize_exact_match, combination_limit, concurrency, query_hashes, id_buff);
+
+ if(id_buff.size() > 1) {
+ std::sort(id_buff.begin(), id_buff.end());
+ id_buff.erase(std::unique( id_buff.begin(), id_buff.end() ), id_buff.end());
+ }
+
+ uint32_t* new_all_result_ids = nullptr;
+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &id_buff[0],
+ id_buff.size(), &new_all_result_ids);
+ delete[] *all_result_ids;
+ *all_result_ids = new_all_result_ids;
}
resume_typo_loop:
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 63ceba4f..09229ed0 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1939,7 +1939,7 @@ TEST_F(CollectionTest, DeletionOfDocumentArrayFields) {
token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10).get();
- ASSERT_EQ(1, res["found"]);
+ ASSERT_EQ(1, res["found"].get<size_t>());
Option<std::string> rem_op = coll1->remove("100");