Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix double locking of collection mutex.

Harpreet Sangara1c694924671
diff --git a/include/index.h b/include/index.h
index e142e6ae..e0875935 100644
--- a/include/index.h
+++ b/include/index.h
@@ -484,21 +484,27 @@ private:
uint32_t*& ids,
size_t& ids_len) const;
- void do_filtering(filter_node_t* const root) const;
+ void do_filtering(filter_node_t* const root, const std::string& collection_name) const;
- void rearranging_recursive_filter (uint32_t*& filter_ids, uint32_t& filter_ids_length, filter_node_t* const root) const;
+ void rearranging_recursive_filter (uint32_t*& filter_ids,
+ uint32_t& filter_ids_length,
+ filter_node_t* const root,
+ const std::string& collection_name) const;
void recursive_filter(uint32_t*& filter_ids,
uint32_t& filter_ids_length,
filter_node_t* const root,
- const bool enable_short_circuit = false) const;
+ const std::string& collection_name) const;
void adaptive_filter(uint32_t*& filter_ids,
uint32_t& filter_ids_length,
filter_node_t* const filter_tree_root,
- const bool enable_short_circuit = false) const;
+ const std::string& collection_name = "") const;
- void get_filter_matches(filter_node_t* const root, std::vector<std::pair<uint32_t, filter_node_t*>>& vec) const;
+ void get_filter_matches(filter_node_t* const root,
+ std::vector<std::pair<uint32_t,
+ filter_node_t*>>& vec,
+ const std::string& collection_name) const;
void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;
@@ -656,7 +662,7 @@ public:
// Public operations
- void run_search(search_args* search_params);
+ void run_search(search_args* search_params, const std::string& collection_name);
void search(std::vector<query_tokens_t>& field_query_tokens, const std::vector<search_field_t>& the_fields,
const text_match_type_t match_type,
@@ -679,7 +685,8 @@ public:
size_t max_candidates, const std::vector<enable_t>& infixes, const size_t max_extra_prefix,
const size_t max_extra_suffix, const size_t facet_query_num_typos,
const bool filter_curated_hits, enable_t split_join_tokens,
- const vector_query_t& vector_query, size_t facet_sample_percent, size_t facet_sample_threshold) const;
+ const vector_query_t& vector_query, size_t facet_sample_percent, size_t facet_sample_threshold,
+ const std::string& collection_name) const;
void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name);
@@ -720,7 +727,8 @@ public:
void do_filtering_with_lock(
uint32_t*& filter_ids,
uint32_t& filter_ids_length,
- filter_node_t* filter_tree_root) const;
+ filter_node_t* filter_tree_root,
+ const std::string& collection_name) const;
void do_reference_filtering_with_lock(std::pair<uint32_t, uint32_t*>& reference_index_ids,
filter_node_t* filter_tree_root,
diff --git a/src/collection.cpp b/src/collection.cpp
index f95c7db4..5181f690 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1448,7 +1448,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
filter_curated_hits, split_join_tokens, vector_query,
facet_sample_percent, facet_sample_threshold);
- index->run_search(search_params);
+ index->run_search(search_params, name);
// for grouping we have to re-aggregate
@@ -2411,7 +2411,7 @@ Option<bool> Collection::get_filter_ids(const std::string & filter_query,
uint32_t* filter_ids = nullptr;
uint32_t filter_ids_len = 0;
- index->do_filtering_with_lock(filter_ids, filter_ids_len, filter_tree_root);
+ index->do_filtering_with_lock(filter_ids, filter_ids_len, filter_tree_root, name);
index_ids.emplace_back(filter_ids_len, filter_ids);
delete filter_tree_root;
diff --git a/src/index.cpp b/src/index.cpp
index 3b9e7bb0..2a6680f8 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1635,7 +1635,7 @@ void Index::numeric_not_equals_filter(num_tree_t* const num_tree,
ids = out;
}
-void Index::do_filtering(filter_node_t* const root) const {
+void Index::do_filtering(filter_node_t* const root, const std::string& collection_name) const {
// auto begin = std::chrono::high_resolution_clock::now();
const filter a_filter = root->filter_exp;
@@ -1646,7 +1646,7 @@ void Index::do_filtering(filter_node_t* const root) const {
auto collection = cm.get_collection(a_filter.referenced_collection_name);
auto op = collection->get_reference_filter_ids(a_filter.field_name,
- cm.get_collection_with_id(collection_id)->get_name(),
+ collection_name,
root->match_index_ids);
if (!op.ok()) {
return;
@@ -1975,26 +1975,29 @@ void Index::do_filtering(filter_node_t* const root) const {
LOG(INFO) << "Time taken for filtering: " << timeMillis << "ms";*/
}
-void Index::get_filter_matches(filter_node_t* const root, std::vector<std::pair<uint32_t, filter_node_t*>>& vec) const {
+void Index::get_filter_matches(filter_node_t* const root,
+ std::vector<std::pair<uint32_t,
+ filter_node_t*>>& vec,
+ const std::string& collection_name) const {
if (root == nullptr) {
return;
}
if (root->isOperator) {
if (root->filter_operator == AND) {
- get_filter_matches(root->left, vec);
- get_filter_matches(root->right, vec);
+ get_filter_matches(root->left, vec, collection_name);
+ get_filter_matches(root->right, vec, collection_name);
} else {
uint32_t *l_filter_ids = nullptr;
uint32_t l_filter_ids_length = 0;
if (root->left != nullptr) {
- rearranging_recursive_filter(l_filter_ids, l_filter_ids_length, root->left);
+ rearranging_recursive_filter(l_filter_ids, l_filter_ids_length, root->left, collection_name);
}
uint32_t *r_filter_ids = nullptr;
uint32_t r_filter_ids_length = 0;
if (root->right != nullptr) {
- rearranging_recursive_filter(r_filter_ids, r_filter_ids_length, root->right);
+ rearranging_recursive_filter(r_filter_ids, r_filter_ids_length, root->right, collection_name);
}
root->match_index_ids.first = ArrayUtils::or_scalar(
@@ -2010,7 +2013,7 @@ void Index::get_filter_matches(filter_node_t* const root, std::vector<std::pair<
return;
}
- do_filtering(root);
+ do_filtering(root, collection_name);
vec.emplace_back(root->match_index_ids.first, root);
}
@@ -2049,9 +2052,12 @@ void evaluate_rearranged_filter_tree(uint32_t*& filter_ids,
}
}
-void Index::rearranging_recursive_filter(uint32_t*& filter_ids, uint32_t& filter_ids_length, filter_node_t* const root) const {
+void Index::rearranging_recursive_filter(uint32_t*& filter_ids,
+ uint32_t& filter_ids_length,
+ filter_node_t* const root,
+ const std::string& collection_name) const {
std::vector<std::pair<uint32_t, filter_node_t*>> vec;
- get_filter_matches(root, vec);
+ get_filter_matches(root, vec, collection_name);
std::sort(vec.begin(), vec.end(),
[](const std::pair<uint32_t, filter_node_t*>& lhs, const std::pair<uint32_t, filter_node_t*>& rhs) {
@@ -2068,7 +2074,7 @@ void Index::rearranging_recursive_filter(uint32_t*& filter_ids, uint32_t& filter
void Index::recursive_filter(uint32_t*& filter_ids,
uint32_t& filter_ids_length,
filter_node_t* const root,
- const bool enable_short_circuit) const {
+ const std::string& collection_name) const {
if (root == nullptr) {
return;
}
@@ -2077,15 +2083,13 @@ void Index::recursive_filter(uint32_t*& filter_ids,
uint32_t* l_filter_ids = nullptr;
uint32_t l_filter_ids_length = 0;
if (root->left != nullptr) {
- recursive_filter(l_filter_ids, l_filter_ids_length, root->left,
- enable_short_circuit);
+ recursive_filter(l_filter_ids, l_filter_ids_length, root->left,collection_name);
}
uint32_t* r_filter_ids = nullptr;
uint32_t r_filter_ids_length = 0;
if (root->right != nullptr) {
- recursive_filter(r_filter_ids, r_filter_ids_length, root->right,
- enable_short_circuit);
+ recursive_filter(r_filter_ids, r_filter_ids_length, root->right,collection_name);
}
uint32_t* filtered_results = nullptr;
@@ -2106,7 +2110,7 @@ void Index::recursive_filter(uint32_t*& filter_ids,
return;
}
- do_filtering(root);
+ do_filtering(root, collection_name);
filter_ids_length = root->match_index_ids.first;
filter_ids = root->match_index_ids.second;
@@ -2117,7 +2121,7 @@ void Index::recursive_filter(uint32_t*& filter_ids,
void Index::adaptive_filter(uint32_t*& filter_ids,
uint32_t& filter_ids_length,
filter_node_t* const filter_tree_root,
- const bool enable_short_circuit) const {
+ const std::string& collection_name) const {
if (filter_tree_root == nullptr) {
return;
}
@@ -2127,24 +2131,25 @@ void Index::adaptive_filter(uint32_t*& filter_ids,
(*filter_tree_root->metrics).and_operator_count > 0 &&
// If there are more || in the filter tree than &&, we'll not gain much by rearranging the filter tree.
((float) (*filter_tree_root->metrics).or_operator_count / (float) (*filter_tree_root->metrics).and_operator_count < 0.5)) {
- rearranging_recursive_filter(filter_ids, filter_ids_length, filter_tree_root);
+ rearranging_recursive_filter(filter_ids, filter_ids_length, filter_tree_root, collection_name);
} else {
- recursive_filter(filter_ids, filter_ids_length, filter_tree_root, false);
+ recursive_filter(filter_ids, filter_ids_length, filter_tree_root, collection_name);
}
}
void Index::do_filtering_with_lock(uint32_t*& filter_ids,
uint32_t& filter_ids_length,
- filter_node_t* filter_tree_root) const {
+ filter_node_t* filter_tree_root,
+ const std::string& collection_name) const {
std::shared_lock lock(mutex);
- adaptive_filter(filter_ids, filter_ids_length, filter_tree_root, false);
+ adaptive_filter(filter_ids, filter_ids_length, filter_tree_root, collection_name);
}
void Index::do_reference_filtering_with_lock(std::pair<uint32_t, uint32_t*>& reference_index_ids,
filter_node_t* filter_tree_root,
const std::string& reference_helper_field_name) const {
std::shared_lock lock(mutex);
- adaptive_filter(reference_index_ids.second, reference_index_ids.first, filter_tree_root, false);
+ adaptive_filter(reference_index_ids.second, reference_index_ids.first, filter_tree_root);
std::vector<uint32> vector;
vector.reserve(reference_index_ids.first);
@@ -2160,7 +2165,7 @@ void Index::do_reference_filtering_with_lock(std::pair<uint32_t, uint32_t*>& ref
std::copy(vector.begin(), vector.end(), reference_index_ids.second);
}
-void Index::run_search(search_args* search_params) {
+void Index::run_search(search_args* search_params, const std::string& collection_name) {
search(search_params->field_query_tokens,
search_params->search_fields,
search_params->match_type,
@@ -2193,7 +2198,8 @@ void Index::run_search(search_args* search_params) {
search_params->split_join_tokens,
search_params->vector_query,
search_params->facet_sample_percent,
- search_params->facet_sample_threshold);
+ search_params->facet_sample_threshold,
+ collection_name);
}
void Index::collate_included_ids(const std::vector<token_t>& q_included_tokens,
@@ -2643,7 +2649,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
const size_t max_extra_suffix, const size_t facet_query_num_typos,
const bool filter_curated_hits, const enable_t split_join_tokens,
const vector_query_t& vector_query,
- size_t facet_sample_percent, size_t facet_sample_threshold) const {
+ size_t facet_sample_percent, size_t facet_sample_threshold,
+ const std::string& collection_name) const {
// process the filters
@@ -2652,7 +2659,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
std::shared_lock lock(mutex);
- adaptive_filter(filter_ids, filter_ids_length, filter_tree_root, true);
+ adaptive_filter(filter_ids, filter_ids_length, filter_tree_root, collection_name);
if (filter_tree_root != nullptr && filter_ids_length == 0) {
delete [] filter_ids;
@@ -4748,7 +4755,7 @@ void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint
field_values[i] = &seq_id_sentinel_value;
} else if (sort_fields_std[i].name == sort_field_const::eval) {
field_values[i] = &eval_sentinel_value;
- adaptive_filter(sort_fields_std[i].eval.ids, sort_fields_std[i].eval.size, sort_fields_std[i].eval.filter_tree_root, true);
+ adaptive_filter(sort_fields_std[i].eval.ids, sort_fields_std[i].eval.size, sort_fields_std[i].eval.filter_tree_root);
} else if (search_schema.count(sort_fields_std[i].name) != 0 && search_schema.at(sort_fields_std[i].name).sort) {
if (search_schema.at(sort_fields_std[i].name).type == field_types::GEOPOINT_ARRAY) {
geopoint_indices.push_back(i);
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index 7d45523a..ab1936d2 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -299,7 +299,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {
collectionManager.drop_collection("Products");
}
-TEST_F(CollectionJoinTest, FilterByReferenceField) {
+TEST_F(CollectionJoinTest, FilterByReferenceField_SingleMatch) {
auto schema_json =
R"({
"name": "Products",
@@ -404,4 +404,149 @@ TEST_F(CollectionJoinTest, FilterByReferenceField) {
ASSERT_EQ(1, result["found"].get<size_t>());
ASSERT_EQ(1, result["hits"].size());
ASSERT_EQ("soap", result["hits"][0]["document"]["product_name"].get<std::string>());
+
+// collectionManager.drop_collection("Customers");
+// collectionManager.drop_collection("Products");
}
+
+TEST_F(CollectionJoinTest, FilterByReferenceField_MultipleMatch) {
+ auto schema_json =
+ R"({
+ "name": "Users",
+ "fields": [
+ {"name": "user_id", "type": "string"},
+ {"name": "user_name", "type": "string"}
+ ]
+ })"_json;
+ std::vector<nlohmann::json> documents = {
+ R"({
+ "user_id": "user_a",
+ "user_name": "Roshan"
+ })"_json,
+ R"({
+ "user_id": "user_b",
+ "user_name": "Ruby"
+ })"_json,
+ R"({
+ "user_id": "user_c",
+ "user_name": "Joe"
+ })"_json,
+ R"({
+ "user_id": "user_d",
+ "user_name": "Aby"
+ })"_json
+ };
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "Repos",
+ "fields": [
+ {"name": "repo_id", "type": "string"},
+ {"name": "repo_content", "type": "string"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "repo_id": "repo_a",
+ "repo_content": "body1"
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "repo_content": "body2"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "repo_content": "body3"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "Links",
+ "fields": [
+ {"name": "repo_id", "type": "string", "reference": "Repos.repo_id"},
+ {"name": "user_id", "type": "string", "reference": "Users.user_id"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "repo_id": "repo_a",
+ "user_id": "user_b"
+ })"_json,
+ R"({
+ "repo_id": "repo_a",
+ "user_id": "user_c"
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "user_id": "user_a"
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "user_id": "user_b"
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "user_id": "user_d"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_a"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_b"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_c"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_d"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ auto coll = collectionManager.get_collection("Users");
+
+ // Search for users linked to repo_b
+ auto result = coll->search("R", {"user_name"}, "$Links(repo_id:=repo_b)", {}, {}, {0},
+ 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD).get();
+
+ ASSERT_EQ(2, result["found"].get<size_t>());
+ ASSERT_EQ(2, result["hits"].size());
+ ASSERT_EQ("user_b", result["hits"][0]["document"]["user_id"].get<std::string>());
+ ASSERT_EQ("user_a", result["hits"][1]["document"]["user_id"].get<std::string>());
+
+// collectionManager.drop_collection("Users");
+// collectionManager.drop_collection("Repos");
+// collectionManager.drop_collection("Links");
+}
\ No newline at end of file