Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Tweak locking for filtering.

Kishore Nallan4c1dc3a4955c
diff --git a/include/index.h b/include/index.h
index b2aad2fd..fd0b1ae1 100644
--- a/include/index.h
+++ b/include/index.h
@@ -500,6 +500,8 @@ private:
size_t concurrency,
std::set<uint64>& query_hashes) const;
+ void do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length, const std::vector<filter>& filters) const;
+
void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;
@@ -694,7 +696,8 @@ public:
art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);
- void do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length, const std::vector<filter>& filters) const;
+ void do_filtering_with_lock(uint32_t*& filter_ids, uint32_t& filter_ids_length,
+ const std::vector<filter>& filters) const;
void refresh_schemas(const std::vector<field>& new_fields);
diff --git a/src/collection.cpp b/src/collection.cpp
index ed5bec6b..7c28f7d1 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1346,7 +1346,7 @@ Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,
uint32_t* filter_ids = nullptr;
uint32_t filter_ids_len = 0;
- index->do_filtering(filter_ids, filter_ids_len, filters);
+ index->do_filtering_with_lock(filter_ids, filter_ids_len, filters);
index_ids.emplace_back(filter_ids_len, filter_ids);
return Option<bool>(true);
diff --git a/src/index.cpp b/src/index.cpp
index 6b09752e..93163ea1 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1247,8 +1247,6 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,
const std::vector<filter>& filters) const {
//auto begin = std::chrono::high_resolution_clock::now();
- std::shared_lock lock(mutex);
-
for(size_t i = 0; i < filters.size(); i++) {
const filter & a_filter = filters[i];
@@ -1620,6 +1618,13 @@ void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,
LOG(INFO) << "Time taken for filtering: " << timeMillis << "ms";*/
}
+
+void Index::do_filtering_with_lock(uint32_t*& filter_ids, uint32_t& filter_ids_length,
+ const std::vector<filter>& filters) const {
+ std::shared_lock lock(mutex);
+ do_filtering(filter_ids, filter_ids_length, filters);
+}
+
void Index::run_search(search_args* search_params) {
search(search_params->field_query_tokens,
search_params->search_fields,
@@ -2086,12 +2091,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,
process_filter_overrides(filter_overrides, field_query_tokens, token_order, filters);
- lock.unlock();
-
do_filtering(filter_ids, filter_ids_length, filters);
- lock.lock();
-
// Order of `fields` are used to sort results
//auto begin = std::chrono::high_resolution_clock::now();
uint32_t* all_result_ids = nullptr;