Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix eval + hybrid search.
Kishore Nallan9a6ef3cbe482
diff --git a/src/index.cpp b/src/index.cppindex 5d51487c..d7a5ead6 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -3367,11 +3367,22 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons}vec_results.emplace_back(seq_id, vec_dist_score);}-++ // iteration needs to happen on sorted sequence ID but score wise sort needed for compute rank fusionstd::sort(vec_results.begin(), vec_results.end(), [](const auto& a, const auto& b) {return a.second < b.second;});+ std::unordered_map<uint32_t, uint32_t> seq_id_to_rank;++ for(size_t vec_index = 0; vec_index < vec_results.size(); vec_index++) {+ seq_id_to_rank.emplace(vec_results[vec_index].first, vec_index);+ }++ std::sort(vec_results.begin(), vec_results.end(), [](const auto& a, const auto& b) {+ return a.first < b.first;+ });+std::vector<KV*> kvs;if(group_limit != 0) {for(auto& kv_map : topster->group_kv_map) {@@ -3385,8 +3396,6 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, constopster->sort();}--// Reciprocal rank fusion// Score is sum of (1 / rank_of_document) * WEIGHT from each list (text match and vector search)auto size = (group_limit != 0) ? kvs.size() : topster->size;@@ -3444,7 +3453,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consfound_kv->text_match_score = found_kv->scores[found_kv->match_score_index];int64_t match_score = float_to_int64_t((int64_t_to_float(found_kv->scores[found_kv->match_score_index])) +- ((1.0 / (res_index + 1)) * VECTOR_SEARCH_WEIGHT));+ ((1.0 / (seq_id_to_rank[seq_id] + 1)) * VECTOR_SEARCH_WEIGHT));int64_t match_score_index = -1;int64_t scores[3] = {0};@@ -3466,7 +3475,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons// Result has been found only in vector search: we have to add it to both KV and result_ids// (1 / rank_of_document) * WEIGHT)int64_t scores[3] = {0};- int64_t match_score = float_to_int64_t((1.0 / (res_index + 1)) * VECTOR_SEARCH_WEIGHT);+ int64_t match_score = float_to_int64_t((1.0 / (seq_id_to_rank[seq_id] + 1)) * VECTOR_SEARCH_WEIGHT);int64_t match_score_index = -1;auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex 452e8a4a..42009f85 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -1423,6 +1423,54 @@ TEST_F(CollectionVectorTest, HybridSearchSortByGeopoint) {ASSERT_EQ("butterfly", search_res["hits"][2]["document"]["name"].get<std::string>());}+TEST_F(CollectionVectorTest, HybridSearchWithEvalSort) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "name", "type": "string", "facet": true},+ {"name": "category", "type": "string", "facet": true},+ {"name": "vec", "type": "float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;++ EmbedderManager::set_model_dir("/tmp/typesense_test/models");+ Collection* coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["name"] = "Apple Fruit";+ doc["category"] = "Fresh";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["id"] = "1";+ doc["name"] = "Apple";+ doc["category"] = "Phone";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["id"] = "2";+ doc["name"] = "Apple Pie";+ doc["category"] = "Notebook";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ std::vector<sort_by> sort_fields;+ CollectionManager::parse_sort_by_str("_eval([(category:Fresh):3,(category:Notebook):2,(category:Phone):1]):desc", sort_fields);++ auto results_op = coll1->search("apple", {"name", "vec"}, "", {"name"}, sort_fields, {0}, 20, 1, FREQUENCY, {true},+ Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2);+ ASSERT_EQ(true, results_op.ok());+ ASSERT_EQ(3, results_op.get()["found"].get<size_t>());+ ASSERT_EQ(3, results_op.get()["hits"].size());++ ASSERT_EQ("0", results_op.get()["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("2", results_op.get()["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results_op.get()["hits"][2]["document"]["id"].get<std::string>());+}TEST_F(CollectionVectorTest, EmbedFromOptionalNullField) {nlohmann::json schema = R"({