Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Sort by reference fields in case of many-to-many relation.

Harpreet Sangar08c893b3bd76
diff --git a/include/index.h b/include/index.h
index 0e45dffc..8574afe8 100644
--- a/include/index.h
+++ b/include/index.h
@@ -453,7 +453,7 @@ private:
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
const std::vector<size_t>& geopoint_indices,
std::set<uint64>& query_hashes,
- std::vector<uint32_t>& id_buff) const;
+ std::vector<uint32_t>& id_buff, const std::string& collection_name = "") const;
void search_candidates(const uint8_t & field_id,
bool field_is_array,
@@ -709,7 +709,8 @@ public:
const size_t concurrency,
const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices) const;
+ const std::vector<size_t>& geopoint_indices,
+ const std::string& collection_name = "") const;
void search_infix(const std::string& query, const std::string& field_name, std::vector<uint32_t>& ids,
size_t max_extra_prefix, size_t max_extra_suffix) const;
@@ -762,7 +763,8 @@ public:
const std::vector<uint32_t>& curated_ids_sorted,
const std::unordered_set<uint32_t>& excluded_group_ids,
uint32_t*& all_result_ids, size_t& all_result_ids_len,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const;
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ const std::string& collection_name = "") const;
[[nodiscard]] Option<bool> do_synonym_search(const std::vector<search_field_t>& the_fields,
const text_match_type_t match_type,
@@ -791,7 +793,8 @@ public:
const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
const std::vector<size_t>& geopoint_indices,
- tsl::htrie_map<char, token_leaf>& qtoken_set) const;
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ const std::string& collection_name = "") const;
Option<bool> do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,
std::vector<query_tokens_t>& field_query_tokens,
@@ -811,7 +814,7 @@ public:
const std::unordered_set<uint32_t>& excluded_group_ids,
Topster* curated_topster,
const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- bool is_wildcard_query) const;
+ bool is_wildcard_query, const std::string& collection_name = "") const;
[[nodiscard]] Option<bool> fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
const std::vector<token_t>& query_tokens,
@@ -842,7 +845,8 @@ public:
int syn_orig_num_tokens,
const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices) const;
+ const std::vector<size_t>& geopoint_indices,
+ const std::string& collection_name = "") const;
void find_across_fields(const token_t& previous_token,
const std::string& previous_token_str,
@@ -880,8 +884,8 @@ public:
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
const std::vector<size_t>& geopoint_indices,
std::vector<uint32_t>& id_buff,
- uint32_t*& all_result_ids,
- size_t& all_result_ids_len) const;
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const std::string& collection_name = "") const;
void
search_fields(const std::vector<filter>& filters,
@@ -918,7 +922,8 @@ public:
size_t filter_index,
int64_t max_field_match_score,
int64_t* scores,
- int64_t& match_score_index, float vector_distance = 0) const;
+ int64_t& match_score_index, float vector_distance = 0,
+ const std::string& collection_name = "") const;
void process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
const std::vector<uint32_t>& excluded_ids, const std::vector<std::string>& group_by_fields,
diff --git a/src/collection.cpp b/src/collection.cpp
index e749e75b..f5489f05 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -4419,7 +4419,7 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
auto& cm = CollectionManager::get_instance();
auto ref_collection = cm.get_collection(ref_collection_name);
if (ref_collection == nullptr) {
- return Option<bool>(400, "Referenced collection `" + ref_collection_name + "` in include_fields not found.");
+ return Option<bool>(400, "Referenced collection `" + ref_collection_name + "` in `include_fields` not found.");
}
auto const joined_on_ref_collection = reference_filter_results.count(ref_collection_name) > 0,
diff --git a/src/index.cpp b/src/index.cpp
index 8da8276d..a3930721 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1471,7 +1471,7 @@ Option<bool> Index::search_all_candidates(const size_t num_search_fields,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
const std::vector<size_t>& geopoint_indices,
std::set<uint64>& query_hashes,
- std::vector<uint32_t>& id_buff) const {
+ std::vector<uint32_t>& id_buff, const std::string& collection_name) const {
/*if(!token_candidates_vec.empty()) {
LOG(INFO) << "Prefix candidates size: " << token_candidates_vec.back().candidates.size();
@@ -1520,7 +1520,8 @@ Option<bool> Index::search_all_candidates(const size_t num_search_fields,
total_cost, syn_orig_num_tokens,
exclude_token_ids, exclude_token_ids_size, excluded_group_ids,
sort_order, field_values, geopoint_indices,
- id_buff, all_result_ids, all_result_ids_len);
+ id_buff, all_result_ids, all_result_ids_len,
+ collection_name);
if (!search_across_fields_op.ok()) {
return search_across_fields_op;
}
@@ -2282,7 +2283,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
filter_result_iterator, all_result_ids, all_result_ids_len,
groups_processed, curated_ids,
excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
- curated_topster, included_ids_map, is_wildcard_query);
+ curated_topster, included_ids_map, is_wildcard_query,
+ collection_name);
filter_iterator_guard.release();
filter_iterator_guard.reset(filter_result_iterator);
@@ -2446,7 +2448,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,
geopoint_indices, seq_id, references, 0, 0, scores,
- match_score_index, vec_dist_score);
+ match_score_index, vec_dist_score, collection_name);
if (!compute_sort_scores_op.ok()) {
return compute_sort_scores_op;
}
@@ -2480,7 +2482,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
all_result_ids, all_result_ids_len,
filter_result_iterator, concurrency,
- sort_order, field_values, geopoint_indices);
+ sort_order, field_values, geopoint_indices, collection_name);
if (!search_wildcard_op.ok()) {
return search_wildcard_op;
}
@@ -2539,7 +2541,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
prioritize_token_position, query_hashes, token_order, prefixes,
typo_tokens_threshold, exhaustive_search,
max_candidates, min_len_1typo, min_len_2typo,
- syn_orig_num_tokens, sort_order, field_values, geopoint_indices);
+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
+ collection_name);
if (!fuzzy_search_fields_op.ok()) {
return fuzzy_search_fields_op;
}
@@ -2582,7 +2585,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
prioritize_token_position, query_hashes, token_order,
prefixes, typo_tokens_threshold, exhaustive_search,
max_candidates, min_len_1typo, min_len_2typo,
- syn_orig_num_tokens, sort_order, field_values, geopoint_indices);
+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
+ collection_name);
if (!fuzzy_search_fields_op.ok()) {
return fuzzy_search_fields_op;
}
@@ -2600,7 +2604,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
groups_processed, searched_queries, all_result_ids, all_result_ids_len,
filter_result_iterator, query_hashes,
sort_order, field_values, geopoint_indices,
- qtoken_set);
+ qtoken_set, collection_name);
if (!do_synonym_search_op.ok()) {
return do_synonym_search_op;
}
@@ -2662,7 +2666,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
prioritize_exact_match, prioritize_token_position, query_hashes,
token_order, prefixes, typo_tokens_threshold,
exhaustive_search, max_candidates, min_len_1typo,
- min_len_2typo, -1, sort_order, field_values, geopoint_indices);
+ min_len_2typo, -1, sort_order, field_values, geopoint_indices,
+ collection_name);
if (!fuzzy_search_fields_op.ok()) {
return fuzzy_search_fields_op;
}
@@ -2682,7 +2687,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
topster, filter_result_iterator,
sort_order, field_values, geopoint_indices,
curated_ids_sorted, excluded_group_ids,
- all_result_ids, all_result_ids_len, groups_processed);
+ all_result_ids, all_result_ids_len, groups_processed,
+ collection_name);
if (!do_infix_search_op.ok()) {
return do_infix_search_op;
}
@@ -2781,7 +2787,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,
geopoint_indices, doc_id, references, 0,
match_score, scores, match_score_index,
- vec_result.second);
+ vec_result.second, collection_name);
if (!compute_sort_scores_op.ok()) {
return compute_sort_scores_op;
}
@@ -2800,7 +2806,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,
geopoint_indices, doc_id, references, 0,
match_score, scores, match_score_index,
- vec_result.second);
+ vec_result.second, collection_name);
if (!compute_sort_scores_op.ok()) {
return compute_sort_scores_op;
}
@@ -3138,7 +3144,8 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_f
int syn_orig_num_tokens,
const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices) const {
+ const std::vector<size_t>& geopoint_indices,
+ const std::string& collection_name) const {
// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens
@@ -3379,7 +3386,7 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_f
num_typos, prefixes, prioritize_exact_match, prioritize_token_position,
exhaustive_search, max_candidates,
syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
- query_hashes, id_buff);
+ query_hashes, id_buff, collection_name);
if (!search_all_candidates_op.ok()) {
return search_all_candidates_op;
}
@@ -3547,7 +3554,8 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_token
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
const std::vector<size_t>& geopoint_indices,
std::vector<uint32_t>& id_buff,
- uint32_t*& all_result_ids, size_t& all_result_ids_len) const {
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const std::string& collection_name) const {
std::vector<art_leaf*> query_suggestion;
@@ -3740,7 +3748,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_token
auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,
seq_id, references, filter_index, best_field_match_score,
- scores, match_score_index);
+ scores, match_score_index, 0, collection_name);
if (!compute_sort_scores_op.ok()) {
status = Option<bool>(compute_sort_scores_op.code(), compute_sort_scores_op.error());
return;
@@ -3856,7 +3864,8 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
const std::vector<size_t>& geopoint_indices,
uint32_t seq_id, const std::map<basic_string<char>, reference_filter_result_t>& references,
size_t filter_index, int64_t max_field_match_score, int64_t* scores,
- int64_t& match_score_index, float vector_distance) const {
+ int64_t& match_score_index, float vector_distance,
+ const std::string& collection_name) const {
int64_t geopoint_distances[3];
@@ -3917,19 +3926,84 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
// In case of reference sort_by, we need to get the sort score of the reference doc id.
if (!sort_fields[0].reference_collection_name.empty()) {
auto& sort_field = sort_fields[0];
- if (references.count(sort_field.reference_collection_name) == 0 ||
- references.at(sort_field.reference_collection_name).count == 0) {
- return Option<bool>(400, "No references found to sort by on `" +
- sort_field.reference_collection_name + "." + sort_field.name + "`.");
- }
+ auto const& ref_collection_name = sort_field.reference_collection_name;
+ auto const& multiple_references_error_message = "Multiple references found to sort by on `" +
+ ref_collection_name + "." + sort_field.name + "`.";
+ auto const& no_references_error_message = "No references found to sort by on `" +
+ ref_collection_name + "." + sort_field.name + "`.";
+
+ // Joined on ref collection
+ if (references.count(ref_collection_name) > 0) {
+ if (references.at(ref_collection_name).count == 1) {
+ seq_id = references.at(ref_collection_name).docs[0];
+ } else {
+ return Option<bool>(400, references.at(ref_collection_name).count > 1 ?
+ multiple_references_error_message : no_references_error_message);
+ }
+ } else {
+ auto& cm = CollectionManager::get_instance();
+ auto ref_collection = cm.get_collection(ref_collection_name);
+ if (ref_collection == nullptr) {
+ return Option<bool>(400, "Referenced collection `" + ref_collection_name +
+ "` in `sort_by` not found.");
+ }
- auto& reference_result = references.at(sort_field.reference_collection_name);
- if (reference_result.count > 1) {
- return Option<bool>(400, "Multiple references found to sort by on `" +
- sort_field.reference_collection_name + "." + sort_field.name + "`.");
- }
+ // Current collection has a reference.
+ if (ref_collection->is_referenced_in(collection_name)) {
+ auto get_reference_field_op = ref_collection->get_reference_field(collection_name);
+ if (!get_reference_field_op.ok()) {
+ return Option<bool>(get_reference_field_op.code(), get_reference_field_op.error());
+ }
+ auto const& field_name = get_reference_field_op.get();
+ if (sort_index.count(field_name) == 0 || sort_index.at(field_name)->count(seq_id) == 0) {
+ return Option<bool>(400, "Could not find a reference for doc " + std::to_string(seq_id));
+ }
+
+ seq_id = sort_index.at(field_name)->at(seq_id);
+ }
+ // Joined collection has a reference
+ else {
+ std::string joined_coll_having_reference;
+ for (const auto &reference: references) {
+ if (ref_collection->is_referenced_in(reference.first)) {
+ joined_coll_having_reference = reference.first;
+ break;
+ }
+ }
+
+ if (joined_coll_having_reference.empty()) {
+ return Option<bool>(400, no_references_error_message);
+ }
+
+ auto joined_collection = cm.get_collection(joined_coll_having_reference);
+ if (joined_collection == nullptr) {
+ return Option<bool>(400, "Referenced collection `" + joined_coll_having_reference +
+ "` in `sort_by` not found.");
+ }
+
+ auto reference_field_name_op = ref_collection->get_reference_field(joined_coll_having_reference);
+ if (!reference_field_name_op.ok()) {
+ return Option<bool>(reference_field_name_op.code(), reference_field_name_op.error());
+ }
+
+ auto const& reference_field_name = reference_field_name_op.get();
+ auto const& reference = references.at(joined_coll_having_reference);
+ auto const& count = reference.count;
+
+ if (count == 1) {
+ auto op = joined_collection->get_sort_indexed_field_value(reference_field_name,
+ reference.docs[0]);
+ if (!op.ok()) {
+ return Option<bool>(op.code(), op.error());
+ }
- seq_id = reference_result.docs[0];
+ seq_id = op.get();
+ } else {
+ return Option<bool>(400, count > 1 ? multiple_references_error_message :
+ no_references_error_message);
+ }
+ }
+ }
}
if (field_values[0] == &text_match_sentinel_value) {
@@ -4005,19 +4079,84 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
// In case of reference sort_by, we need to get the sort score of the reference doc id.
if (!sort_fields[1].reference_collection_name.empty()) {
auto& sort_field = sort_fields[1];
- if (references.count(sort_field.reference_collection_name) == 0 ||
- references.at(sort_field.reference_collection_name).count == 0) {
- return Option<bool>(400, "No references found to sort by on `" +
- sort_field.reference_collection_name + "." + sort_field.name + "`.");
- }
+ auto const& ref_collection_name = sort_field.reference_collection_name;
+ auto const& multiple_references_error_message = "Multiple references found to sort by on `" +
+ ref_collection_name + "." + sort_field.name + "`.";
+ auto const& no_references_error_message = "No references found to sort by on `" +
+ ref_collection_name + "." + sort_field.name + "`.";
+
+ // Joined on ref collection
+ if (references.count(ref_collection_name) > 0) {
+ if (references.at(ref_collection_name).count == 1) {
+ seq_id = references.at(ref_collection_name).docs[0];
+ } else {
+ return Option<bool>(400, references.at(ref_collection_name).count > 1 ?
+ multiple_references_error_message : no_references_error_message);
+ }
+ } else {
+ auto& cm = CollectionManager::get_instance();
+ auto ref_collection = cm.get_collection(ref_collection_name);
+ if (ref_collection == nullptr) {
+ return Option<bool>(400, "Referenced collection `" + ref_collection_name +
+ "` in `sort_by` not found.");
+ }
- auto& reference_result = references.at(sort_field.reference_collection_name);
- if (reference_result.count > 1) {
- return Option<bool>(400, "Multiple references found to sort by on `" +
- sort_field.reference_collection_name + "." + sort_field.name + "`.");
- }
+ // Current collection has a reference.
+ if (ref_collection->is_referenced_in(collection_name)) {
+ auto get_reference_field_op = ref_collection->get_reference_field(collection_name);
+ if (!get_reference_field_op.ok()) {
+ return Option<bool>(get_reference_field_op.code(), get_reference_field_op.error());
+ }
+ auto const& field_name = get_reference_field_op.get();
+ if (sort_index.count(field_name) == 0 || sort_index.at(field_name)->count(seq_id) == 0) {
+ return Option<bool>(400, "Could not find a reference for doc " + std::to_string(seq_id));
+ }
- seq_id = reference_result.docs[0];
+ seq_id = sort_index.at(field_name)->at(seq_id);
+ }
+ // Joined collection has a reference
+ else {
+ std::string joined_coll_having_reference;
+ for (const auto &reference: references) {
+ if (ref_collection->is_referenced_in(reference.first)) {
+ joined_coll_having_reference = reference.first;
+ break;
+ }
+ }
+
+ if (joined_coll_having_reference.empty()) {
+ return Option<bool>(400, no_references_error_message);
+ }
+
+ auto joined_collection = cm.get_collection(joined_coll_having_reference);
+ if (joined_collection == nullptr) {
+ return Option<bool>(400, "Referenced collection `" + joined_coll_having_reference +
+ "` in `sort_by` not found.");
+ }
+
+ auto reference_field_name_op = ref_collection->get_reference_field(joined_coll_having_reference);
+ if (!reference_field_name_op.ok()) {
+ return Option<bool>(reference_field_name_op.code(), reference_field_name_op.error());
+ }
+
+ auto const& reference_field_name = reference_field_name_op.get();
+ auto const& reference = references.at(joined_coll_having_reference);
+ auto const& count = reference.count;
+
+ if (count == 1) {
+ auto op = joined_collection->get_sort_indexed_field_value(reference_field_name,
+ reference.docs[0]);
+ if (!op.ok()) {
+ return Option<bool>(op.code(), op.error());
+ }
+
+ seq_id = op.get();
+ } else {
+ return Option<bool>(400, count > 1 ? multiple_references_error_message :
+ no_references_error_message);
+ }
+ }
+ }
}
if (field_values[1] == &text_match_sentinel_value) {
@@ -4089,19 +4228,84 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
// In case of reference sort_by, we need to get the sort score of the reference doc id.
if (!sort_fields[2].reference_collection_name.empty()) {
auto& sort_field = sort_fields[2];
- if (references.count(sort_field.reference_collection_name) == 0 ||
- references.at(sort_field.reference_collection_name).count == 0) {
- return Option<bool>(400, "No references found to sort by on `" +
- sort_field.reference_collection_name + "." + sort_field.name + "`.");
- }
+ auto const& ref_collection_name = sort_field.reference_collection_name;
+ auto const& multiple_references_error_message = "Multiple references found to sort by on `" +
+ ref_collection_name + "." + sort_field.name + "`.";
+ auto const& no_references_error_message = "No references found to sort by on `" +
+ ref_collection_name + "." + sort_field.name + "`.";
+
+ // Joined on ref collection
+ if (references.count(ref_collection_name) > 0) {
+ if (references.at(ref_collection_name).count == 1) {
+ seq_id = references.at(ref_collection_name).docs[0];
+ } else {
+ return Option<bool>(400, references.at(ref_collection_name).count > 1 ?
+ multiple_references_error_message : no_references_error_message);
+ }
+ } else {
+ auto& cm = CollectionManager::get_instance();
+ auto ref_collection = cm.get_collection(ref_collection_name);
+ if (ref_collection == nullptr) {
+ return Option<bool>(400, "Referenced collection `" + ref_collection_name +
+ "` in `sort_by` not found.");
+ }
- auto& reference_result = references.at(sort_field.reference_collection_name);
- if (reference_result.count > 1) {
- return Option<bool>(400, "Multiple references found to sort by on `" +
- sort_field.reference_collection_name + "." + sort_field.name + "`.");
- }
+ // Current collection has a reference.
+ if (ref_collection->is_referenced_in(collection_name)) {
+ auto get_reference_field_op = ref_collection->get_reference_field(collection_name);
+ if (!get_reference_field_op.ok()) {
+ return Option<bool>(get_reference_field_op.code(), get_reference_field_op.error());
+ }
+ auto const& field_name = get_reference_field_op.get();
+ if (sort_index.count(field_name) == 0 || sort_index.at(field_name)->count(seq_id) == 0) {
+ return Option<bool>(400, "Could not find a reference for doc " + std::to_string(seq_id));
+ }
+
+ seq_id = sort_index.at(field_name)->at(seq_id);
+ }
+ // Joined collection has a reference
+ else {
+ std::string joined_coll_having_reference;
+ for (const auto &reference: references) {
+ if (ref_collection->is_referenced_in(reference.first)) {
+ joined_coll_having_reference = reference.first;
+ break;
+ }
+ }
+
+ if (joined_coll_having_reference.empty()) {
+ return Option<bool>(400, no_references_error_message);
+ }
+
+ auto joined_collection = cm.get_collection(joined_coll_having_reference);
+ if (joined_collection == nullptr) {
+ return Option<bool>(400, "Referenced collection `" + joined_coll_having_reference +
+ "` in `sort_by` not found.");
+ }
- seq_id = reference_result.docs[0];
+ auto reference_field_name_op = ref_collection->get_reference_field(joined_coll_having_reference);
+ if (!reference_field_name_op.ok()) {
+ return Option<bool>(reference_field_name_op.code(), reference_field_name_op.error());
+ }
+
+ auto const& reference_field_name = reference_field_name_op.get();
+ auto const& reference = references.at(joined_coll_having_reference);
+ auto const& count = reference.count;
+
+ if (count == 1) {
+ auto op = joined_collection->get_sort_indexed_field_value(reference_field_name,
+ reference.docs[0]);
+ if (!op.ok()) {
+ return Option<bool>(op.code(), op.error());
+ }
+
+ seq_id = op.get();
+ } else {
+ return Option<bool>(400, count > 1 ? multiple_references_error_message :
+ no_references_error_message);
+ }
+ }
+ }
}
if (field_values[2] == &text_match_sentinel_value) {
@@ -4190,7 +4394,7 @@ Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::
const std::unordered_set<uint32_t>& excluded_group_ids,
Topster* curated_topster,
const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- bool is_wildcard_query) const {
+ bool is_wildcard_query, const std::string& collection_name) const {
uint32_t* phrase_result_ids = nullptr;
uint32_t phrase_result_count = 0;
@@ -4312,7 +4516,7 @@ Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::
auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,
seq_id, references, filter_index, match_score, scores,
- match_score_index);
+ match_score_index, 0, collection_name);
if (!compute_sort_scores_op.ok()) {
return compute_sort_scores_op;
}
@@ -4369,7 +4573,8 @@ Option<bool> Index::do_synonym_search(const std::vector<search_field_t>& the_fie
const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
const std::vector<size_t>& geopoint_indices,
- tsl::htrie_map<char, token_leaf>& qtoken_set) const {
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ const std::string& collection_name) const {
for (const auto& syn_tokens : q_pos_synonyms) {
query_hashes.clear();
@@ -4381,7 +4586,8 @@ Option<bool> Index::do_synonym_search(const std::vector<search_field_t>& the_fie
prioritize_exact_match, prioritize_token_position, query_hashes,
token_order, prefixes, typo_tokens_threshold, exhaustive_search,
max_candidates, min_len_1typo, min_len_2typo,
- syn_orig_num_tokens, sort_order, field_values, geopoint_indices);
+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
+ collection_name);
if (!fuzzy_search_fields_op.ok()) {
return fuzzy_search_fields_op;
}
@@ -4405,7 +4611,8 @@ Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::v
const std::vector<uint32_t>& curated_ids_sorted,
const std::unordered_set<uint32_t>& excluded_group_ids,
uint32_t*& all_result_ids, size_t& all_result_ids_len,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const {
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ const std::string& collection_name) const {
for(size_t field_id = 0; field_id < num_search_fields; field_id++) {
auto& field_name = the_fields[field_id].name;
@@ -4462,7 +4669,8 @@ Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::v
auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values,
geopoint_indices, seq_id, references,
- filter_index, 100, scores, match_score_index);
+ filter_index, 100, scores, match_score_index,
+ 0, collection_name);
if (!compute_sort_scores_op.ok()) {
return compute_sort_scores_op;
}
@@ -4770,7 +4978,8 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root
const size_t concurrency,
const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices) const {
+ const std::vector<size_t>& geopoint_indices,
+ const std::string& collection_name) const {
filter_result_iterator->compute_result();
auto const& approx_filter_ids_length = filter_result_iterator->approx_filter_ids_length;
@@ -4818,7 +5027,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root
&sort_order, field_values, &geopoint_indices, &plists,
check_for_circuit_break,
batch_result,
- &num_processed, &m_process, &cv_process, &compute_sort_score_status]() {
+ &num_processed, &m_process, &cv_process, &compute_sort_score_status, collection_name]() {
search_begin_us = parent_search_begin;
search_stop_us = parent_search_stop_ms;
@@ -4843,7 +5052,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root
auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,
seq_id, references, filter_index, 100, scores,
- match_score_index);
+ match_score_index, 0, collection_name);
if (!compute_sort_scores_op.ok()) {
compute_sort_score_status = new Option<bool>(compute_sort_scores_op.code(), compute_sort_scores_op.error());
break;
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index cffa7345..97ed9392 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -1150,7 +1150,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
req_params["include_fields"] = "$foo(bar)";
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_FALSE(search_op.ok());
- ASSERT_EQ("Referenced collection `foo` in include_fields not found.", search_op.error());
+ ASSERT_EQ("Referenced collection `foo` in `include_fields` not found.", search_op.error());
req_params = {
{"collection", "Products"},
@@ -2308,6 +2308,248 @@ TEST_F(CollectionJoinTest, SortByReference) {
ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));
ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));
+ // Reference sort_by without join
+ req_params = {
+ {"collection", "Customers"},
+ {"q", "*"},
+ {"filter_by", "customer_name:= [Joe, Dan] && product_price:<100"},
+ {"include_fields", "$Products(product_name), product_price"},
+ {"sort_by", "$Products(product_name:desc)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("product_name"));
+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ("shampoo", res_obj["hits"][1]["document"].at("product_name"));
+ ASSERT_EQ(75, res_obj["hits"][1]["document"].at("product_price"));
+
+ req_params = {
+ {"collection", "Customers"},
+ {"q", "*"},
+ {"filter_by", "customer_name:= [Joe, Dan] && product_price:<100"},
+ {"include_fields", "$Products(product_name), product_price"},
+ {"sort_by", "$Products(product_name:asc)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ("shampoo", res_obj["hits"][0]["document"].at("product_name"));
+ ASSERT_EQ(75, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ("soap", res_obj["hits"][1]["document"].at("product_name"));
+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));
+
+ schema_json =
+ R"({
+ "name": "Users",
+ "fields": [
+ {"name": "user_id", "type": "string"},
+ {"name": "user_name", "type": "string"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "user_id": "user_a",
+ "user_name": "Roshan"
+ })"_json,
+ R"({
+ "user_id": "user_b",
+ "user_name": "Ruby"
+ })"_json,
+ R"({
+ "user_id": "user_c",
+ "user_name": "Joe"
+ })"_json,
+ R"({
+ "user_id": "user_d",
+ "user_name": "Aby"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "Repos",
+ "fields": [
+ {"name": "repo_id", "type": "string"},
+ {"name": "repo_content", "type": "string"},
+ {"name": "repo_stars", "type": "int32"},
+ {"name": "repo_is_private", "type": "bool"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "repo_id": "repo_a",
+ "repo_content": "body1",
+ "repo_stars": 431,
+ "repo_is_private": true
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "repo_content": "body2",
+ "repo_stars": 4562,
+ "repo_is_private": false
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "repo_content": "body3",
+ "repo_stars": 945,
+ "repo_is_private": false
+ })"_json,
+ R"({
+ "repo_id": "repo_d",
+ "repo_content": "body4",
+ "repo_stars": 95,
+ "repo_is_private": true
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "Links",
+ "fields": [
+ {"name": "repo_id", "type": "string", "reference": "Repos.repo_id"},
+ {"name": "user_id", "type": "string", "reference": "Users.user_id"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "repo_id": "repo_a",
+ "user_id": "user_b"
+ })"_json,
+ R"({
+ "repo_id": "repo_a",
+ "user_id": "user_c"
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "user_id": "user_a"
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "user_id": "user_b"
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "user_id": "user_d"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_a"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_b"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_c"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_d"
+ })"_json,
+ R"({
+ "repo_id": "repo_d",
+ "user_id": "user_d"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ req_params = {
+ {"collection", "Users"},
+ {"q", "*"},
+ {"filter_by", "$Links(repo_id:=[repo_a, repo_d])"},
+ {"include_fields", "user_id, user_name, $Repos(repo_content, repo_stars), "},
+ {"exclude_fields", "$Links(*), "},
+ {"sort_by", "$Repos(repo_stars: asc)"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(3, res_obj["found"].get<size_t>());
+ ASSERT_EQ(3, res_obj["hits"].size());
+ ASSERT_EQ(4, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ("user_d", res_obj["hits"][0]["document"].at("user_id"));
+ ASSERT_EQ("Aby", res_obj["hits"][0]["document"].at("user_name"));
+ ASSERT_EQ("body4", res_obj["hits"][0]["document"].at("repo_content"));
+ ASSERT_EQ(95, res_obj["hits"][0]["document"].at("repo_stars"));
+
+ ASSERT_EQ("user_c", res_obj["hits"][1]["document"].at("user_id"));
+ ASSERT_EQ("Joe", res_obj["hits"][1]["document"].at("user_name"));
+ ASSERT_EQ("body1", res_obj["hits"][1]["document"].at("repo_content"));
+ ASSERT_EQ(431, res_obj["hits"][1]["document"].at("repo_stars"));
+
+ ASSERT_EQ("user_b", res_obj["hits"][2]["document"].at("user_id"));
+ ASSERT_EQ("Ruby", res_obj["hits"][2]["document"].at("user_name"));
+ ASSERT_EQ("body1", res_obj["hits"][2]["document"].at("repo_content"));
+ ASSERT_EQ(431, res_obj["hits"][2]["document"].at("repo_stars"));
+
+ req_params = {
+ {"collection", "Users"},
+ {"q", "*"},
+ {"filter_by", "$Links(repo_id:=[repo_a, repo_d])"},
+ {"include_fields", "user_id, user_name, $Repos(repo_content, repo_stars), "},
+ {"exclude_fields", "$Links(*), "},
+ {"sort_by", "$Repos(repo_stars: desc)"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(3, res_obj["found"].get<size_t>());
+ ASSERT_EQ(3, res_obj["hits"].size());
+ ASSERT_EQ(4, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ("user_c", res_obj["hits"][0]["document"].at("user_id"));
+ ASSERT_EQ("Joe", res_obj["hits"][0]["document"].at("user_name"));
+ ASSERT_EQ("body1", res_obj["hits"][0]["document"].at("repo_content"));
+ ASSERT_EQ(431, res_obj["hits"][0]["document"].at("repo_stars"));
+
+ ASSERT_EQ("user_b", res_obj["hits"][1]["document"].at("user_id"));
+ ASSERT_EQ("Ruby", res_obj["hits"][1]["document"].at("user_name"));
+ ASSERT_EQ("body1", res_obj["hits"][1]["document"].at("repo_content"));
+ ASSERT_EQ(431, res_obj["hits"][1]["document"].at("repo_stars"));
+
+ ASSERT_EQ("user_d", res_obj["hits"][2]["document"].at("user_id"));
+ ASSERT_EQ("Aby", res_obj["hits"][2]["document"].at("user_name"));
+ ASSERT_EQ("body4", res_obj["hits"][2]["document"].at("repo_content"));
+ ASSERT_EQ(95, res_obj["hits"][2]["document"].at("repo_stars"));
+
// Multiple references - Wildcard search
req_params = {
{"collection", "Products"},