Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix reference _eval. (#1859)
Harpreet Sangare2685f4f8f4b
diff --git a/src/collection.cpp b/src/collection.cppindex 61225ea9..d40e4a48 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1190,7 +1190,8 @@ Option<bool> Collection::validate_and_standardize_sort_fields_with_lock(const stconst size_t remote_embedding_num_tries) const {std::shared_lock lock(mutex);return validate_and_standardize_sort_fields(sort_fields, sort_fields_std, is_wildcard_query, is_vector_query,- query, is_group_by_query, remote_embedding_timeout_ms, true);+ query, is_group_by_query, remote_embedding_timeout_ms, remote_embedding_num_tries,+ true);}Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,diff --git a/src/index.cpp b/src/index.cppindex a9d0df67..6095ed06 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -4941,9 +4941,10 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,// avoiding loopif (sort_fields.size() > 0) {auto reference_found = true;+ auto const& is_reference_sort = !sort_fields[0].reference_collection_name.empty();// In case of reference sort_by, we need to get the sort score of the reference doc id.- if (!sort_fields[0].reference_collection_name.empty()) {+ if (is_reference_sort) {auto const& ref_compute_op = ref_compute_sort_scores(sort_fields[0], seq_id, ref_seq_id, reference_found,references, collection_name);if (!ref_compute_op.ok()) {@@ -4959,7 +4960,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,} else if(field_values[0] == &geo_sentinel_value) {scores[0] = geopoint_distances[0];} else if(field_values[0] == &str_sentinel_value) {- if (sort_fields[0].reference_collection_name.empty()) {+ if (!is_reference_sort) {scores[0] = str_sort_index.at(sort_fields[0].name)->rank(seq_id);} else if (!reference_found) {scores[0] = adi_tree_t::NOT_FOUND;@@ -4995,15 +4996,18 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,uint32_t index = 0;auto const& eval = sort_fields[0].eval;for (; index < count; index++) {- auto& filter_index = filter_indexes[index];+ // ref_seq_id(s) can be unordered.+ uint32_t ref_filter_index = 0;+ auto& filter_index = is_reference_sort ? ref_filter_index : filter_indexes[index];auto const& eval_ids = eval.eval_ids_vec[index];auto const& eval_ids_count = eval.eval_ids_count_vec[index];if (filter_index == 0 || filter_index < eval_ids_count) {// Returns iterator to the first element that is >= to value or last if no such element is found.- filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -+ auto const& id = is_reference_sort ? ref_seq_id : seq_id;+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, id) -eval_ids;- if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {+ if (filter_index < eval_ids_count && eval_ids[filter_index] == id) {filter_index++;found = true;break;@@ -5027,8 +5031,8 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,// do nothing}} else {- if (sort_fields[0].reference_collection_name.empty() || reference_found) {- auto it = field_values[0]->find(sort_fields[0].reference_collection_name.empty() ? seq_id : ref_seq_id);+ if (!is_reference_sort || reference_found) {+ auto it = field_values[0]->find(is_reference_sort ? ref_seq_id : seq_id);scores[0] = (it == field_values[0]->end()) ? default_score : it->second;} else {scores[0] = default_score;@@ -5050,9 +5054,10 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,if(sort_fields.size() > 1) {auto reference_found = true;+ auto const& is_reference_sort = !sort_fields[1].reference_collection_name.empty();// In case of reference sort_by, we need to get the sort score of the reference doc id.- if (!sort_fields[1].reference_collection_name.empty()) {+ if (is_reference_sort) {auto const& ref_compute_op = ref_compute_sort_scores(sort_fields[1], seq_id, ref_seq_id, reference_found,references, collection_name);if (!ref_compute_op.ok()) {@@ -5068,7 +5073,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,} else if(field_values[1] == &geo_sentinel_value) {scores[1] = geopoint_distances[1];} else if(field_values[1] == &str_sentinel_value) {- if (sort_fields[1].reference_collection_name.empty()) {+ if (!is_reference_sort) {scores[1] = str_sort_index.at(sort_fields[1].name)->rank(seq_id);} else if (!reference_found) {scores[1] = adi_tree_t::NOT_FOUND;@@ -5104,15 +5109,18 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,uint32_t index = 0;auto const& eval = sort_fields[1].eval;for (; index < count; index++) {- auto& filter_index = filter_indexes[index];+ // ref_seq_id(s) can be unordered.+ uint32_t ref_filter_index = 0;+ auto& filter_index = is_reference_sort ? ref_filter_index : filter_indexes[index];auto const& eval_ids = eval.eval_ids_vec[index];auto const& eval_ids_count = eval.eval_ids_count_vec[index];if (filter_index == 0 || filter_index < eval_ids_count) {// Returns iterator to the first element that is >= to value or last if no such element is found.- filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -+ auto const& id = is_reference_sort ? ref_seq_id : seq_id;+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, id) -eval_ids;- if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {+ if (filter_index < eval_ids_count && eval_ids[filter_index] == id) {filter_index++;found = true;break;@@ -5121,6 +5129,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,}scores[1] = found ? eval.scores[index] : 0;+ LOG(INFO) << "seq_id: " << seq_id << " ref_seq_id: " << ref_seq_id << " score: " << scores[1] << " index: " << index;} else if(field_values[1] == &vector_distance_sentinel_value) {scores[1] = float_to_int64_t(vector_distance);} else if(field_values[1] == &vector_query_sentinel_value) {@@ -5137,8 +5146,8 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,}} else {- if (sort_fields[1].reference_collection_name.empty() || reference_found) {- auto it = field_values[1]->find(sort_fields[1].reference_collection_name.empty() ? seq_id : ref_seq_id);+ if (!is_reference_sort || reference_found) {+ auto it = field_values[1]->find(is_reference_sort ? ref_seq_id : seq_id);scores[1] = (it == field_values[1]->end()) ? default_score : it->second;} else {scores[1] = default_score;@@ -5157,9 +5166,10 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,if(sort_fields.size() > 2) {auto reference_found = true;+ auto const& is_reference_sort = !sort_fields[2].reference_collection_name.empty();// In case of reference sort_by, we need to get the sort score of the reference doc id.- if (!sort_fields[2].reference_collection_name.empty()) {+ if (is_reference_sort) {auto const& ref_compute_op = ref_compute_sort_scores(sort_fields[2], seq_id, ref_seq_id, reference_found,references, collection_name);if (!ref_compute_op.ok()) {@@ -5175,7 +5185,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,} else if(field_values[2] == &geo_sentinel_value) {scores[2] = geopoint_distances[2];} else if(field_values[2] == &str_sentinel_value) {- if (sort_fields[2].reference_collection_name.empty()) {+ if (!is_reference_sort) {scores[2] = str_sort_index.at(sort_fields[2].name)->rank(seq_id);} else if (!reference_found) {scores[2] = adi_tree_t::NOT_FOUND;@@ -5211,15 +5221,18 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,uint32_t index = 0;auto const& eval = sort_fields[2].eval;for (; index < count; index++) {- auto& filter_index = filter_indexes[index];+ // ref_seq_id(s) can be unordered.+ uint32_t ref_filter_index = 0;+ auto& filter_index = is_reference_sort ? ref_filter_index : filter_indexes[index];auto const& eval_ids = eval.eval_ids_vec[index];auto const& eval_ids_count = eval.eval_ids_count_vec[index];if (filter_index == 0 || filter_index < eval_ids_count) {// Returns iterator to the first element that is >= to value or last if no such element is found.- filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -+ auto const& id = is_reference_sort ? ref_seq_id : seq_id;+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, id) -eval_ids;- if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {+ if (filter_index < eval_ids_count && eval_ids[filter_index] == id) {filter_index++;found = true;break;@@ -5243,8 +5256,8 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,// do nothing}} else {- if (sort_fields[2].reference_collection_name.empty() || reference_found) {- auto it = field_values[2]->find(sort_fields[2].reference_collection_name.empty() ? seq_id : ref_seq_id);+ if (!is_reference_sort || reference_found) {+ auto it = field_values[2]->find(is_reference_sort ? ref_seq_id : seq_id);scores[2] = (it == field_values[2]->end()) ? default_score : it->second;} else {scores[2] = default_score;diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex ba579a66..6e797cdf 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -5788,6 +5788,122 @@ TEST_F(CollectionJoinTest, SortByReference) {ASSERT_EQ("0", res_obj["hits"][3]["document"].at("id"));ASSERT_EQ("bar", res_obj["hits"][3]["document"]["Structures"].at("name"));ASSERT_EQ(0, res_obj["hits"][3]["document"].count("Ads"));++ schema_json =+ R"({+ "name": "product",+ "fields": [+ {"name": "entity_id", "type": "string"},+ {"name": "name", "type": "string", "sort": true}+ ]+ })"_json;+ documents = {+ R"({"entity_id": "P0", "name": "Generic brand Tablet"})"_json,+ R"({"entity_id": "P1", "name": "Tablet from samsung"})"_json,+ R"({"entity_id": "P2", "name": "Tablet from apple"})"_json,+ R"({"entity_id": "P3", "name": "Tablet from oppo"})"_json,+ R"({"entity_id": "P4", "name": "Tablet from vivo"})"_json,+ R"({"entity_id": "P5", "name": "Phone from samsung"})"_json,+ R"({"entity_id": "P6", "name": "Tablet from xiaomi"})"_json,+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());++ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "stock",+ "fields": [+ {"name": "entity_id", "type": "string", "reference": "product.entity_id"},+ {"name": "store_.*", "type": "bool", "sort": true}+ ]+ })"_json;+ documents = {+ R"({"entity_id": "P0", "store_1": true, "store_2": true})"_json,+ R"({"entity_id": "P1", "store_1": false, "store_2": false})"_json,+ R"({"entity_id": "P2", "store_1": false, "store_2": true})"_json,+ R"({"entity_id": "P4", "store_1": true, "store_2": true})"_json,+ R"({"entity_id": "P6", "store_1": false, "store_2": false})"_json,+ R"({"entity_id": "P3", "store_1": true, "store_2": false})"_json,+ R"({"entity_id": "P5", "store_1": true, "store_2": true})"_json,+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());++ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ req_params = {+ {"collection", "product"},+ {"q", "tablet"},+ {"query_by", "name"},+ {"filter_by", "$stock(id: *)"},+ {"sort_by", "$stock(_eval(store_1:true || store_2:true):desc)"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(6, res_obj["found"].get<size_t>());+ ASSERT_EQ(6, res_obj["hits"].size());+ ASSERT_EQ("4", res_obj["hits"][0]["document"].at("id"));+ ASSERT_EQ("3", res_obj["hits"][1]["document"].at("id"));+ ASSERT_EQ("2", res_obj["hits"][2]["document"].at("id"));+ ASSERT_EQ("0", res_obj["hits"][3]["document"].at("id"));+ ASSERT_EQ("6", res_obj["hits"][4]["document"].at("id"));+ ASSERT_EQ("1", res_obj["hits"][5]["document"].at("id"));++ req_params = {+ {"collection", "product"},+ {"q", "tablet"},+ {"query_by", "name"},+ {"filter_by", "$stock(id: *)"},+ {"sort_by", "$stock(_eval([(store_1:true && store_2:true):3, (store_1:true || store_2:true):2 ]):desc)"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(6, res_obj["found"].get<size_t>());+ ASSERT_EQ(6, res_obj["hits"].size());+ ASSERT_EQ("4", res_obj["hits"][0]["document"].at("id"));+ ASSERT_EQ("0", res_obj["hits"][1]["document"].at("id"));+ ASSERT_EQ("3", res_obj["hits"][2]["document"].at("id"));+ ASSERT_EQ("2", res_obj["hits"][3]["document"].at("id"));+ ASSERT_EQ("6", res_obj["hits"][4]["document"].at("id"));+ ASSERT_EQ("1", res_obj["hits"][5]["document"].at("id"));++ req_params = {+ {"collection", "product"},+ {"q", "tablet"},+ {"query_by", "name"},+ {"filter_by", "$stock(id: *)"},+ {"sort_by", "_text_match:desc, $stock(_eval([(store_1:true && store_2:true):3, (store_1:true || store_2:true):2 ]):desc)"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(6, res_obj["found"].get<size_t>());+ ASSERT_EQ(6, res_obj["hits"].size());+ ASSERT_EQ("4", res_obj["hits"][0]["document"].at("id"));+ ASSERT_EQ("0", res_obj["hits"][1]["document"].at("id"));+ ASSERT_EQ("3", res_obj["hits"][2]["document"].at("id"));+ ASSERT_EQ("2", res_obj["hits"][3]["document"].at("id"));+ ASSERT_EQ("6", res_obj["hits"][4]["document"].at("id"));+ ASSERT_EQ("1", res_obj["hits"][5]["document"].at("id"));}TEST_F(CollectionJoinTest, FilterByReferenceAlias) {