Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Vector query parse bugs (#1937)
Krunal Gandhif96c43e2eee0
diff --git a/include/index.h b/include/index.hindex 522e0e7d..9e02b0c9 100644--- a/include/index.h+++ b/include/index.h@@ -1024,10 +1024,9 @@ public:std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values,const std::vector<size_t>& geopoint_indices, uint32_t seq_id,const std::map<basic_string<char>, reference_filter_result_t>& references,- std::vector<uint32_t>& filter_indexes,- int64_t max_field_match_score,- int64_t* scores,- int64_t& match_score_index, float vector_distance = 0,+ std::vector<uint32_t>& filter_indexes, int64_t max_field_match_score,+ int64_t* scores, int64_t& match_score_index, bool& should_skip,+ float vector_distance = 0,const std::string& collection_name = "") const;void process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,diff --git a/include/vector_query_ops.h b/include/vector_query_ops.hindex 9c78c2be..57a9384b 100644--- a/include/vector_query_ops.h+++ b/include/vector_query_ops.h@@ -3,6 +3,7 @@#include <string>#include <vector>#include "option.h"+#include <limits>class Collection;@@ -10,7 +11,7 @@ struct vector_query_t {std::string field_name;size_t k = 0;size_t flat_search_cutoff = 0;- float distance_threshold = 2.01;+ float distance_threshold = std::numeric_limits<float>::max();std::vector<float> values;uint32_t seq_id = 0;diff --git a/src/index.cpp b/src/index.cppindex 3a05a8e1..cb7c6306 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -3175,15 +3175,20 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consint64_t scores[3] = {0};int64_t match_score_index = -1;+ bool should_skip = false;auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,geopoint_indices, seq_id, references, eval_filter_indexes,- 0, scores, match_score_index, vec_dist_score,+ 0, scores, match_score_index, should_skip, vec_dist_score,collection_name);if (!compute_sort_scores_op.ok()) {return compute_sort_scores_op;}+ if(should_skip) {+ continue;+ }+KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));kv.vector_distance = vec_dist_score;int ret = topster->add(&kv);@@ -3606,15 +3611,20 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons((1.0 / (seq_id_to_rank[seq_id] + 1)) * VECTOR_SEARCH_WEIGHT));int64_t match_score_index = -1;int64_t scores[3] = {0};+ bool should_skip = false;auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,geopoint_indices, seq_id, references, eval_filter_indexes,- match_score, scores, match_score_index,+ match_score, scores, match_score_index, should_skip,vec_result.second, collection_name);if (!compute_sort_scores_op.ok()) {return compute_sort_scores_op;}+ if(should_skip) {+ continue;+ }+for(int i = 0; i < 3; i++) {found_kv->scores[i] = scores[i];}@@ -3627,15 +3637,20 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consint64_t scores[3] = {0};int64_t match_score = float_to_int64_t((1.0 / (seq_id_to_rank[seq_id] + 1)) * VECTOR_SEARCH_WEIGHT);int64_t match_score_index = -1;+ bool should_skip = false;auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,geopoint_indices, seq_id, references, eval_filter_indexes,- match_score, scores, match_score_index,+ match_score, scores, match_score_index, should_skip,vec_result.second, collection_name);if (!compute_sort_scores_op.ok()) {return compute_sort_scores_op;}+ if(should_skip) {+ continue;+ }+uint64_t distinct_id = seq_id;if (group_limit != 0) {distinct_id = 1;@@ -4725,15 +4740,20 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_tokenint64_t scores[3] = {0};int64_t match_score_index = -1;+ bool should_skip = false;auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,seq_id, references, eval_filter_indexes, best_field_match_score,- scores, match_score_index, 0, collection_name);+ scores, match_score_index, should_skip, 0, collection_name);if (!compute_sort_scores_op.ok()) {status = Option<bool>(compute_sort_scores_op.code(), compute_sort_scores_op.error());return;}+ if(should_skip) {+ return;+ }+query_len = std::min<size_t>(15, query_len);// NOTE: `query_len` is total tokens matched across fields.@@ -5034,7 +5054,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,const std::vector<size_t>& geopoint_indices,uint32_t seq_id, const std::map<basic_string<char>, reference_filter_result_t>& references,std::vector<uint32_t>& filter_indexes, int64_t max_field_match_score, int64_t* scores,- int64_t& match_score_index, float vector_distance,+ int64_t& match_score_index, bool& should_skip, float vector_distance,const std::string& collection_name) const {int64_t geopoint_distances[3];@@ -5182,7 +5202,12 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,const auto& values = sort_fields[0].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);const auto& dist_func = sort_fields[0].vector_query.vector_index->space->get_dist_func();float dist = dist_func(sort_fields[0].vector_query.query.values.data(), values.data(), &sort_fields[0].vector_query.vector_index->num_dim);-++ if(dist > sort_fields[0].vector_query.query.distance_threshold) {+ //if computed distance is more then distance_thershold then we wont add that to results+ should_skip = true;+ }+scores[0] = float_to_int64_t(dist);} catch(...) {// probably not found@@ -5295,7 +5320,12 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,const auto& values = sort_fields[1].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);const auto& dist_func = sort_fields[1].vector_query.vector_index->space->get_dist_func();float dist = dist_func(sort_fields[1].vector_query.query.values.data(), values.data(), &sort_fields[1].vector_query.vector_index->num_dim);-++ if(dist > sort_fields[1].vector_query.query.distance_threshold) {+ //if computed distance is more then distance_thershold then we wont add that to results+ should_skip = true;+ }+scores[1] = float_to_int64_t(dist);} catch(...) {// probably not found@@ -5406,7 +5436,12 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,const auto& values = sort_fields[2].vector_query.vector_index->vecdex->getDataByLabel<float>(seq_id);const auto& dist_func = sort_fields[2].vector_query.vector_index->space->get_dist_func();float dist = dist_func(sort_fields[2].vector_query.query.values.data(), values.data(), &sort_fields[2].vector_query.vector_index->num_dim);-++ if(dist > sort_fields[2].vector_query.query.distance_threshold) {+ //if computed distance is more then distance_thershold then we wont add that to results+ should_skip = true;+ }+scores[2] = float_to_int64_t(dist);} catch(...) {// probably not found@@ -5578,14 +5613,19 @@ Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::int64_t match_score = phrase_match_id_scores[seq_id];int64_t scores[3] = {0};int64_t match_score_index = -1;+ bool should_skip = false;auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,seq_id, references, eval_filter_indexes, match_score, scores,- match_score_index, 0, collection_name);+ match_score_index, should_skip, 0, collection_name);if (!compute_sort_scores_op.ok()) {return compute_sort_scores_op;}+ if(should_skip) {+ continue;+ }+uint64_t distinct_id = seq_id;if(group_limit != 0) {distinct_id = 1;@@ -5752,15 +5792,20 @@ Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::vint64_t scores[3] = {0};int64_t match_score_index = -1;+ bool should_skip = false;auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values,geopoint_indices, seq_id, references,eval_filter_indexes, 100, scores, match_score_index,- 0, collection_name);+ should_skip, 0, collection_name);if (!compute_sort_scores_op.ok()) {return compute_sort_scores_op;}+ if(should_skip) {+ continue;+ }+uint64_t distinct_id = seq_id;if(group_limit != 0) {distinct_id = 1;@@ -6179,15 +6224,20 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootint64_t scores[3] = {0};int64_t match_score_index = -1;+ bool should_skip = false;auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,seq_id, references, filter_indexes, 100, scores,- match_score_index, 0, collection_name);+ match_score_index, should_skip, 0, collection_name);if (!compute_sort_scores_op.ok()) {compute_sort_score_status = new Option<bool>(compute_sort_scores_op.code(), compute_sort_scores_op.error());break;}+ if(should_skip) {+ continue;+ }+uint64_t distinct_id = seq_id;if(group_limit != 0) {distinct_id = 1;diff --git a/src/vector_query_ops.cpp b/src/vector_query_ops.cppindex 0a5a865f..9cef88c1 100644--- a/src/vector_query_ops.cpp+++ b/src/vector_query_ops.cpp@@ -165,12 +165,21 @@ Option<bool> VectorQueryOps::parse_vector_query_str(const std::string& vector_qu}if(param_kv[0] == "distance_threshold") {- if(!StringUtils::is_float(param_kv[1]) || std::stof(param_kv[1]) < 0.0 || std::stof(param_kv[1]) > 2.0) {+ auto search_schema = const_cast<Collection*>(coll)->get_schema();+ auto vector_field_it = search_schema.find(vector_query.field_name);++ if(!StringUtils::is_float(param_kv[1])) {+ return Option<bool>(400, "Malformed vector query string: "+ "`distance_threshold` parameter must be a float.");+ }++ auto distance_threshold = std::stof(param_kv[1]);+ if(vector_field_it->vec_dist == cosine && (distance_threshold < 0.0 || distance_threshold > 2.0)) {return Option<bool>(400, "Malformed vector query string: ""`distance_threshold` parameter must be a float between 0.0-2.0.");}- vector_query.distance_threshold = std::stof(param_kv[1]);+ vector_query.distance_threshold = distance_threshold;}if(param_kv[0] == "alpha") {diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex b2475e0d..3316a3ca 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -4945,4 +4945,85 @@ TEST_F(CollectionVectorTest, TestRestoringImages) {coll = collectionManager.get_collection("test").get();ASSERT_EQ(1, coll->get_summary_json()["num_documents"]);+}++TEST_F(CollectionVectorTest, TestDistanceThresholdWithIP) {+ auto schema_json = R"({+ "name": "products",+ "fields":[+ {"name": "name","type": "string"},+ {"name": "rank_score", "type": "float"},+ {"name": "embedding","type": "float[]", "num_dim":5, "optinal":true, "vec_dist": "ip"}+ ],+ "default_sorting_field": "rank_score"+ })"_json;+++ auto coll_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(coll_op.ok());+ auto coll = coll_op.get();++ std::mt19937 rng;+ rng.seed(47);+ std::uniform_real_distribution<> distrib(-1,1);+ std::uniform_int_distribution<>distrib2(0,100);++ nlohmann::json doc;+ for (auto i = 0; i < 5; ++i) {+ std::vector<float> vector(5);+ std::generate(vector.begin(), vector.end(), [&](){ return distrib(rng); });++ doc["name"] = "document_" + std::to_string(i);+ doc["rank_score"] = distrib2(rng);+ doc["embedding"] = vector;+ ASSERT_TRUE(coll->add(doc.dump()).ok());+ }++ //results ids exceeding distance_threshold will be skipped+ std::map<std::string, std::string> req_params = {+ {"collection", "products"},+ {"q", "document"},+ {"query_by", "*"},+ {"sort_by", "_text_match:desc,_vector_query(embedding:([0.11731103425347378, -0.6694758317235057, -0.6211945774857595, -0.27966758971688255, -0.4683744007950299],distance_threshold:1)):asc,rank_score:desc"},+ {"exclude_fields", "embedding"}+ };+ nlohmann::json embedded_params;+ std::string json_res;+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();+ auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ auto res = nlohmann::json::parse(json_res);+++ ASSERT_EQ(2, res["found"].get<size_t>());+ ASSERT_EQ(93, res["hits"][0]["document"]["rank_score"].get<size_t>());+ ASSERT_EQ(0.2189185470342636, res["hits"][0]["vector_distance"].get<float>());+ ASSERT_EQ(51, res["hits"][1]["document"]["rank_score"].get<size_t>());+ ASSERT_EQ(0.7371898889541626, res["hits"][1]["vector_distance"].get<float>());++ //inner product distances should work when distance_threshold is not given+ req_params = {+ {"collection", "products"},+ {"q", "document"},+ {"query_by", "*"},+ {"sort_by", "_text_match:desc,_vector_query(embedding:([-100,-100,-100,-100,-100])):asc,rank_score:desc"},+ {"exclude_fields", "embedding"}+ };+ now_ts = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ res = nlohmann::json::parse(json_res);++ ASSERT_EQ(5, res["found"].get<size_t>());++ ASSERT_EQ("document_1", res["hits"][0]["document"]["name"]);+ ASSERT_EQ(-45.23314666748047, res["hits"][0]["vector_distance"].get<float>());+ ASSERT_EQ("document_2", res["hits"][1]["document"]["name"]);+ ASSERT_EQ(-38.66290283203125, res["hits"][1]["vector_distance"].get<float>());+ ASSERT_EQ("document_4", res["hits"][2]["document"]["name"]);+ ASSERT_EQ(-36.0988655090332, res["hits"][2]["vector_distance"].get<float>());+ ASSERT_EQ("document_3", res["hits"][3]["document"]["name"]);+ ASSERT_EQ(9.637892723083496, res["hits"][3]["vector_distance"].get<float>());+ ASSERT_EQ("document_0", res["hits"][4]["document"]["name"]);+ ASSERT_EQ(288.0364685058594, res["hits"][4]["vector_distance"].get<float>());}\ No newline at end of file